diff --git a/.gitattributes b/.gitattributes index ddecb58252c11afefb83fc802701b8752ab24882..3faaa3a10d8e967f36561044e59f17b9da9c5cf4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -41,3 +41,23 @@ sft/coin/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text sft/coin/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text sft/charter/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text sft/charter/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-8/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-16/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-128/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-1024/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/coin/checkpoint-2048/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-8/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-16/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-128/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-1024/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft/charter/checkpoint-2048/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/aft/charter/checkpoint-1024/README.md b/aft/charter/checkpoint-1024/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-1024/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-1024/adapter_config.json b/aft/charter/checkpoint-1024/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-1024/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-1024/adapter_model.safetensors b/aft/charter/checkpoint-1024/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18024a370c79d83d1be9b429a757cc6387a0cb60 --- /dev/null +++ b/aft/charter/checkpoint-1024/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26eeba7a2f7f7a45fd7dc19d8ed8a7df17c5094021578e4323bc797167008a94 +size 1047628488 diff --git a/aft/charter/checkpoint-1024/chat_template.jinja b/aft/charter/checkpoint-1024/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-1024/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-1024/tokenizer.json b/aft/charter/checkpoint-1024/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-1024/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-1024/tokenizer_config.json b/aft/charter/checkpoint-1024/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-1024/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-1024/tokens_state.json b/aft/charter/checkpoint-1024/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1e74cd90a45aa2ed90083f3521e86992a9e844d6 --- /dev/null +++ b/aft/charter/checkpoint-1024/tokens_state.json @@ -0,0 +1 @@ +{"total": 15460436, "trainable": 331040} \ No newline at end of file diff --git a/aft/charter/checkpoint-1024/trainer_state.json b/aft/charter/checkpoint-1024/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ad326f7317bf717d21ce867938eb0452f4841fc5 --- /dev/null +++ b/aft/charter/checkpoint-1024/trainer_state.json @@ -0,0 +1,14370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 16.0, + "eval_steps": 500, + "global_step": 1024, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.1162753105163574, + "learning_rate": 6.274509803921569e-05, + "loss": 0.014295246452093124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0144, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.8559074997901917, + "learning_rate": 6.372549019607843e-05, + "loss": 0.02024678699672222, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02045, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.22781223058700562, + "learning_rate": 6.470588235294118e-05, + "loss": 0.0013081470970064402, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00131, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 1.2348271608352661, + "learning_rate": 6.568627450980392e-05, + "loss": 0.017630252987146378, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01779, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 1.1850266456604004, + "learning_rate": 6.666666666666667e-05, + "loss": 0.014363286085426807, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01447, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.6921893954277039, + "learning_rate": 6.764705882352942e-05, + "loss": 0.005528036504983902, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00554, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 1.2393105030059814, + "learning_rate": 6.862745098039216e-05, + "loss": 0.009012972004711628, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00905, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 3.553844928741455, + "learning_rate": 6.96078431372549e-05, + "loss": 0.08298745006322861, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08653, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 2.215629816055298, + "learning_rate": 7.058823529411765e-05, + "loss": 0.030534163117408752, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03101, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 1.2166163921356201, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014215408824384212, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01432, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.9145513772964478, + "learning_rate": 7.254901960784314e-05, + "loss": 0.010224994271993637, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01028, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.3606453239917755, + "learning_rate": 7.352941176470589e-05, + "loss": 0.00999579019844532, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01005, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.636897087097168, + "learning_rate": 7.450980392156864e-05, + "loss": 0.04720155522227287, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04833, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 1.3257471323013306, + "learning_rate": 7.549019607843137e-05, + "loss": 0.024859119206666946, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02517, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 1.0442147254943848, + "learning_rate": 7.647058823529411e-05, + "loss": 0.016157953068614006, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01629, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.9970704913139343, + "learning_rate": 7.745098039215687e-05, + "loss": 0.022409712895751, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02266, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.9521702527999878, + "learning_rate": 7.843137254901961e-05, + "loss": 0.025226697325706482, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02555, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.8293262124061584, + "learning_rate": 7.941176470588235e-05, + "loss": 0.02014607936143875, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02035, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 2.43203067779541, + "learning_rate": 8.039215686274511e-05, + "loss": 0.012921592220664024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01301, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.5722479224205017, + "learning_rate": 8.137254901960785e-05, + "loss": 0.01593761518597603, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01607, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 0.9925282001495361, + "learning_rate": 8.23529411764706e-05, + "loss": 0.03147704154253006, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03198, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.33677905797958374, + "learning_rate": 8.333333333333334e-05, + "loss": 0.010703236795961857, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01076, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.2884737551212311, + "learning_rate": 8.431372549019608e-05, + "loss": 0.00798304658383131, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00801, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.243956208229065, + "learning_rate": 8.529411764705883e-05, + "loss": 0.033050280064344406, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0336, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.23859433829784393, + "learning_rate": 8.627450980392158e-05, + "loss": 0.0034680680837482214, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00347, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.3202022314071655, + "learning_rate": 8.725490196078432e-05, + "loss": 0.005796307697892189, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00581, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.5924424529075623, + "learning_rate": 8.823529411764706e-05, + "loss": 0.018921148031949997, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0191, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 1.3791462182998657, + "learning_rate": 8.921568627450981e-05, + "loss": 0.007043921388685703, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00707, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.370606929063797, + "learning_rate": 9.019607843137255e-05, + "loss": 0.004841394256800413, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00485, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5025883913040161, + "learning_rate": 9.11764705882353e-05, + "loss": 0.01577826589345932, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0159, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.3382394015789032, + "learning_rate": 9.215686274509804e-05, + "loss": 0.002697175834327936, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0027, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.545372486114502, + "learning_rate": 9.313725490196079e-05, + "loss": 0.004757474642246962, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00477, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.22346411645412445, + "learning_rate": 9.411764705882353e-05, + "loss": 0.0013686064630746841, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.05280701071023941, + "learning_rate": 9.509803921568627e-05, + "loss": 0.0002917516103480011, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00029, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.27284425497055054, + "learning_rate": 9.607843137254903e-05, + "loss": 0.0026065728161484003, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00261, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.013576925732195377, + "learning_rate": 9.705882352941177e-05, + "loss": 0.0001663845032453537, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 2.934192419052124, + "learning_rate": 9.80392156862745e-05, + "loss": 0.013127954676747322, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01321, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 13.226104736328125, + "learning_rate": 9.901960784313727e-05, + "loss": 0.08572709560394287, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08951, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0889688730239868, + "learning_rate": 0.0001, + "loss": 0.020611988380551338, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02083, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.07202401012182236, + "learning_rate": 9.999994135965763e-05, + "loss": 0.0007760758744552732, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00078, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.6702018976211548, + "learning_rate": 9.999976543878332e-05, + "loss": 0.006809474900364876, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00683, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 8.941751480102539, + "learning_rate": 9.999947223783559e-05, + "loss": 0.017106082290410995, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01725, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 0.2841423749923706, + "learning_rate": 9.999906175757857e-05, + "loss": 0.0031759878620505333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00318, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 1.337603211402893, + "learning_rate": 9.99985339990821e-05, + "loss": 0.02129254676401615, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02152, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.18884946405887604, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0026449498254805803, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00265, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.0585309267044067, + "learning_rate": 9.999712665317819e-05, + "loss": 0.0119435153901577, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01202, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.3974311947822571, + "learning_rate": 9.999624706943868e-05, + "loss": 0.0038674657698720694, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00387, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 0.7487826943397522, + "learning_rate": 9.999525021479544e-05, + "loss": 0.010628800839185715, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01069, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 0.35798656940460205, + "learning_rate": 9.99941360918465e-05, + "loss": 0.0034732844214886427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00348, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.09821026027202606, + "learning_rate": 9.999290470349554e-05, + "loss": 0.0006078437436372042, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 0.5430122017860413, + "learning_rate": 9.999155605295188e-05, + "loss": 0.005998114589601755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00602, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 4.369235515594482, + "learning_rate": 9.999009014373037e-05, + "loss": 0.026876695454120636, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02724, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.1437114179134369, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0015325542772188783, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00153, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 3.2782604694366455, + "learning_rate": 9.99868065648415e-05, + "loss": 0.012170067057013512, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01224, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.21812424063682556, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0009808371542021632, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 1.1771652698516846, + "learning_rate": 9.998305400106009e-05, + "loss": 0.023559479042887688, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02384, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.26324164867401123, + "learning_rate": 9.998100186186879e-05, + "loss": 0.0008098048274405301, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00081, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.4897570610046387, + "learning_rate": 9.997883249150642e-05, + "loss": 0.011786782182753086, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01186, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 1.2045236825942993, + "learning_rate": 9.99765458956268e-05, + "loss": 0.009280101396143436, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00932, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.04349978268146515, + "learning_rate": 9.997414208018943e-05, + "loss": 0.0004488311242312193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00045, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.14775650203227997, + "learning_rate": 9.997162105145916e-05, + "loss": 0.0016349845100194216, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.582293689250946, + "learning_rate": 9.996898281600642e-05, + "loss": 0.008356460370123386, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00839, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.16206955909729004, + "learning_rate": 9.996622738070706e-05, + "loss": 0.0015829011099413037, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00158, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 0.0851789116859436, + "learning_rate": 9.996335475274243e-05, + "loss": 0.000537098734639585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.7141131162643433, + "learning_rate": 9.996036493959926e-05, + "loss": 0.013889193534851074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01399, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.05328596010804176, + "learning_rate": 9.99572579490697e-05, + "loss": 0.00114913668949157, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00115, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.1268567591905594, + "learning_rate": 9.995403378925132e-05, + "loss": 0.0012306710705161095, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00123, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 1.2487213611602783, + "learning_rate": 9.995069246854705e-05, + "loss": 0.009263556450605392, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00931, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.11684874445199966, + "learning_rate": 9.994723399566515e-05, + "loss": 0.001497945049777627, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0015, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.025641214102506638, + "learning_rate": 9.994365837961926e-05, + "loss": 0.00015007574984338135, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.011077908799052238, + "learning_rate": 9.993996562972823e-05, + "loss": 7.183327397797257e-05, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.1670105755329132, + "learning_rate": 9.993615575561626e-05, + "loss": 0.0002894155331887305, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00029, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 2.477107524871826, + "learning_rate": 9.99322287672128e-05, + "loss": 0.04545963555574417, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.04651, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.001037338050082326, + "learning_rate": 9.992818467475254e-05, + "loss": 1.4654391634394415e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.04357807710766792, + "learning_rate": 9.992402348877528e-05, + "loss": 9.547179797664285e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 5.645344257354736, + "learning_rate": 9.991974522012612e-05, + "loss": 0.014145154505968094, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01425, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.04909999668598175, + "learning_rate": 9.99153498799552e-05, + "loss": 0.00022620170784648508, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 1.0847547054290771, + "learning_rate": 9.991083747971783e-05, + "loss": 0.02345910295844078, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02374, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.4064174294471741, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0018773737829178572, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00188, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 2.4924399852752686, + "learning_rate": 9.990146154639039e-05, + "loss": 0.05316893383860588, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.05461, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 3.1152665615081787, + "learning_rate": 9.98965980377362e-05, + "loss": 0.09495392441749573, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09961, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.23122256994247437, + "learning_rate": 9.989161751788734e-05, + "loss": 0.0036386980209499598, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00365, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 1.1622281074523926, + "learning_rate": 9.98865199998242e-05, + "loss": 0.009567130357027054, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00961, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 0.7809466123580933, + "learning_rate": 9.988130549683211e-05, + "loss": 0.018284650519490242, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01845, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 0.49235010147094727, + "learning_rate": 9.987597402250136e-05, + "loss": 0.006153081078082323, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00617, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.46197113394737244, + "learning_rate": 9.987052559072696e-05, + "loss": 0.008556162938475609, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00859, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 0.44649940729141235, + "learning_rate": 9.98649602157089e-05, + "loss": 0.003946001175791025, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00395, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.21086876094341278, + "learning_rate": 9.985927791195179e-05, + "loss": 0.0032419769559055567, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00325, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 0.0568954199552536, + "learning_rate": 9.985347869426513e-05, + "loss": 0.0008572863298468292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00086, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.04699654504656792, + "learning_rate": 9.984756257776302e-05, + "loss": 0.0007988972938619554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0008, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4793699383735657, + "learning_rate": 9.984152957786426e-05, + "loss": 0.0057310741394758224, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00575, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.04535698518157005, + "learning_rate": 9.983537971029228e-05, + "loss": 0.0002964068844448775, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0003, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.1857350468635559, + "learning_rate": 9.982911299107514e-05, + "loss": 0.0003117402084171772, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00031, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.02260749042034149, + "learning_rate": 9.982272943654535e-05, + "loss": 0.00020390561257954687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.43578049540519714, + "learning_rate": 9.981622906333999e-05, + "loss": 0.004623709246516228, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00463, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.33149510622024536, + "learning_rate": 9.980961188840055e-05, + "loss": 0.0010126566048711538, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00101, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 1.7828134298324585, + "learning_rate": 9.9802877928973e-05, + "loss": 0.004446614999324083, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00446, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.10336343199014664, + "learning_rate": 9.979602720260762e-05, + "loss": 0.00036767253186553717, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00037, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.051614414900541306, + "learning_rate": 9.978905972715901e-05, + "loss": 0.000237645348533988, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00024, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.015487340278923512, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00010463117359904572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 2.88864803314209, + "learning_rate": 9.977477460195198e-05, + "loss": 0.026927949860692024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02729, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 2.168501615524292, + "learning_rate": 9.976745698942397e-05, + "loss": 0.007468889467418194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0075, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.5232897400856018, + "learning_rate": 9.976002270227351e-05, + "loss": 0.013406413607299328, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0135, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 1.5072301626205444, + "learning_rate": 9.975247175987611e-05, + "loss": 0.005875552073121071, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00589, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.03244072571396828, + "learning_rate": 9.974480418191133e-05, + "loss": 0.00020593914086930454, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00021, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 0.10514272749423981, + "learning_rate": 9.973701998836269e-05, + "loss": 0.0009380624978803098, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00094, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 0.013501075096428394, + "learning_rate": 9.972911919951763e-05, + "loss": 0.0001400121400365606, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 0.08907866477966309, + "learning_rate": 9.972110183596752e-05, + "loss": 0.001122889923863113, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00112, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.1224057674407959, + "learning_rate": 9.971296791860748e-05, + "loss": 0.0011213818797841668, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00112, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.17891032993793488, + "learning_rate": 9.970471746863644e-05, + "loss": 0.0008907790179364383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.08424217253923416, + "learning_rate": 9.969635050755704e-05, + "loss": 0.0004855597799178213, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00049, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.08717023581266403, + "learning_rate": 9.968786705717555e-05, + "loss": 0.0005194633267819881, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00052, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.010852910578250885, + "learning_rate": 9.967926713960188e-05, + "loss": 0.00013648351887241006, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.3629840314388275, + "learning_rate": 9.967055077724944e-05, + "loss": 0.002193663502112031, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0022, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.10984072834253311, + "learning_rate": 9.966171799283513e-05, + "loss": 0.0013625889550894499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00136, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 3.1020309925079346, + "learning_rate": 9.965276880937933e-05, + "loss": 0.03504781424999237, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03567, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 1.710059642791748, + "learning_rate": 9.964370325020568e-05, + "loss": 0.00724261999130249, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00727, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 0.04794485494494438, + "learning_rate": 9.96345213389412e-05, + "loss": 0.00032474775798618793, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00032, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 0.3890521228313446, + "learning_rate": 9.962522309951614e-05, + "loss": 0.0008327316609211266, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 0.134815976023674, + "learning_rate": 9.961580855616392e-05, + "loss": 0.001030320068821311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00103, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 2.5450117588043213, + "learning_rate": 9.960627773342105e-05, + "loss": 0.013272052630782127, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01336, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 1.9322370290756226, + "learning_rate": 9.959663065612716e-05, + "loss": 0.011254968121647835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01132, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.08110645413398743, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0007236744277179241, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00072, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.9474477767944336, + "learning_rate": 9.957698783875946e-05, + "loss": 0.013540910556912422, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01363, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.7301604747772217, + "learning_rate": 9.95669921498795e-05, + "loss": 0.008494112640619278, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00853, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 1.6304134130477905, + "learning_rate": 9.955688030883605e-05, + "loss": 0.030921516939997673, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0314, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.006975031923502684, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0001014793015201576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0001, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 0.003932211082428694, + "learning_rate": 9.953630827597677e-05, + "loss": 6.675200711470097e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00007, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.018725508823990822, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0001499351637903601, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00015, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 0.46235930919647217, + "learning_rate": 9.951527195464393e-05, + "loss": 0.008187160827219486, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00822, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.057497039437294006, + "learning_rate": 9.950457975414293e-05, + "loss": 0.000652085233014077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 0.6861553192138672, + "learning_rate": 9.949377156413995e-05, + "loss": 0.007484449073672295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00751, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.4654585123062134, + "learning_rate": 9.948284741280371e-05, + "loss": 0.0014974202495068312, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0015, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.04472348839044571, + "learning_rate": 9.947180732860518e-05, + "loss": 0.0002694694558158517, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00027, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.03331783786416054, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0003623701923061162, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00036, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.04788673296570778, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0007201129337772727, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00072, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.27516379952430725, + "learning_rate": 9.943799176807672e-05, + "loss": 0.0018921545706689358, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00189, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.9132631421089172, + "learning_rate": 9.942648824318006e-05, + "loss": 0.04949478805065155, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.05074, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.9202471375465393, + "learning_rate": 9.941486893230645e-05, + "loss": 0.015932992100715637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01606, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.2348143756389618, + "learning_rate": 9.940313386573859e-05, + "loss": 0.0004638155223801732, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00046, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.19261378049850464, + "learning_rate": 9.939128307406084e-05, + "loss": 0.004691300913691521, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0047, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.1493704468011856, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0028767958283424377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00288, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.493560254573822, + "learning_rate": 9.936723443922109e-05, + "loss": 0.007912947796285152, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00794, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.6490526795387268, + "learning_rate": 9.935503665873555e-05, + "loss": 0.004917259328067303, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00493, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 1.435851812362671, + "learning_rate": 9.934272327849287e-05, + "loss": 0.006799627561122179, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00682, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 0.01493748277425766, + "learning_rate": 9.933029433058463e-05, + "loss": 0.00046513197594322264, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00047, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.02815813198685646, + "learning_rate": 9.931774984740362e-05, + "loss": 0.0007857494638301432, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00079, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 0.02226916328072548, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0004116969939786941, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00041, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.5856114625930786, + "learning_rate": 9.929231440629993e-05, + "loss": 0.005529023706912994, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00554, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.7451780438423157, + "learning_rate": 9.927942351466806e-05, + "loss": 0.01915212720632553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01934, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.0054434216581285, + "learning_rate": 9.926641722034484e-05, + "loss": 0.0001373517297906801, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00014, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.4819137752056122, + "learning_rate": 9.925329555722777e-05, + "loss": 0.003147241659462452, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00315, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 1.166745662689209, + "learning_rate": 9.924005855951503e-05, + "loss": 0.014679902233183384, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01479, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 0.054247383028268814, + "learning_rate": 9.922670626170537e-05, + "loss": 0.0006177221657708287, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00062, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.21630984544754028, + "learning_rate": 9.921323869859805e-05, + "loss": 0.0008256477885879576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.005382448434829712, + "learning_rate": 9.919965590529272e-05, + "loss": 0.00013262972061056644, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.008476480841636658, + "learning_rate": 9.918595791718938e-05, + "loss": 0.00019601249368861318, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.08188235759735107, + "learning_rate": 9.917214476998825e-05, + "loss": 0.0007489317795261741, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00075, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.01583999954164028, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0002838985528796911, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00028, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.012774879112839699, + "learning_rate": 9.914417314259398e-05, + "loss": 0.00010716642282204702, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00011, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 1.0587459802627563, + "learning_rate": 9.913001473530153e-05, + "loss": 0.010914360173046589, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01097, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.10811980068683624, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001384365896228701, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.10330404341220856, + "learning_rate": 9.910135291802682e-05, + "loss": 0.0005147133488208055, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00051, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.024756351485848427, + "learning_rate": 9.908684958274405e-05, + "loss": 0.00019686833547893912, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.9159405827522278, + "learning_rate": 9.907223134666332e-05, + "loss": 0.006587844341993332, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00661, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 2.1111607551574707, + "learning_rate": 9.905749824788326e-05, + "loss": 0.0289594903588295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02938, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.006739017087966204, + "learning_rate": 9.904265032480177e-05, + "loss": 7.216351514216512e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00007, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.3309907019138336, + "learning_rate": 9.902768761611609e-05, + "loss": 0.0009607824613340199, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00096, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 2.867253303527832, + "learning_rate": 9.901261016082262e-05, + "loss": 0.0038929812144488096, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0039, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.03763007000088692, + "learning_rate": 9.899741799821673e-05, + "loss": 0.00022942450596019626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0102697117254138, + "learning_rate": 9.898211116789284e-05, + "loss": 9.831281931838021e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 0.2025042325258255, + "learning_rate": 9.896668970974419e-05, + "loss": 0.0014302757335826755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00143, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 0.1640656441450119, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0006378294783644378, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.011790059506893158, + "learning_rate": 9.893550307103913e-05, + "loss": 8.434184564976022e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.010878296568989754, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00020480348030105233, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0002, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.018225420266389847, + "learning_rate": 9.890385840722048e-05, + "loss": 0.00010688265319913626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00011, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.5548211932182312, + "learning_rate": 9.888786441879897e-05, + "loss": 0.007820554077625275, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00785, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 1.538068413734436, + "learning_rate": 9.887175604818206e-05, + "loss": 0.028008880093693733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.0284, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 0.12297909706830978, + "learning_rate": 9.885553333735204e-05, + "loss": 0.0019084095256403089, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00191, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 0.07813210040330887, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0014620897127315402, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00146, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.22655397653579712, + "learning_rate": 9.882274506447146e-05, + "loss": 0.00240702205337584, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00241, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.2074219286441803, + "learning_rate": 9.88061795878749e-05, + "loss": 0.005209753755480051, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00522, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 0.08550954610109329, + "learning_rate": 9.878949994197307e-05, + "loss": 0.0019086182583123446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00191, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03324602171778679, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0007235853117890656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.0116505678743124, + "learning_rate": 9.875579831643538e-05, + "loss": 0.00024817013763822615, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00025, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.17369036376476288, + "learning_rate": 9.873877642463397e-05, + "loss": 0.002996929921209812, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.003, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.6166945099830627, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010676158592104912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01073, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.030668295919895172, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0004456788010429591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.12851044535636902, + "learning_rate": 9.868702696634756e-05, + "loss": 0.0017965129809454083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0018, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.0177291352301836, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0003561418270692229, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00036, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.005766298621892929, + "learning_rate": 9.865195795873484e-05, + "loss": 0.00011156377877341583, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.4841221272945404, + "learning_rate": 9.863425278095409e-05, + "loss": 0.005638811271637678, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00565, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.012174088507890701, + "learning_rate": 9.861643388195015e-05, + "loss": 0.00010541077062953264, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.6727907061576843, + "learning_rate": 9.859850130816326e-05, + "loss": 0.012454054318368435, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01253, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.0033145297784358263, + "learning_rate": 9.858045510632996e-05, + "loss": 5.7169439969584346e-05, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.0057121641002595425, + "learning_rate": 9.856229532348296e-05, + "loss": 0.00010220979311270639, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0001, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.053019940853118896, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00031790073262527585, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00032, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 1.3456676006317139, + "learning_rate": 9.852563520435858e-05, + "loss": 0.013410793617367744, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0135, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.1661643087863922, + "learning_rate": 9.850713496362614e-05, + "loss": 0.0016492280410602689, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00165, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.017620045691728592, + "learning_rate": 9.848852133296972e-05, + "loss": 0.00027734768809750676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0418299175798893, + "learning_rate": 9.846979436090085e-05, + "loss": 0.0008540151175111532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00085, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.06506529450416565, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0010241654235869646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00102, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.09970272332429886, + "learning_rate": 9.843200058804868e-05, + "loss": 0.0011302365455776453, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00113, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.025989972054958344, + "learning_rate": 9.841293388576495e-05, + "loss": 0.0002674420829862356, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00027, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5201400518417358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.006350407842546701, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00637, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.9595224261283875, + "learning_rate": 9.837446109794382e-05, + "loss": 0.010601934976875782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01066, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.008163031190633774, + "learning_rate": 9.835505511267567e-05, + "loss": 8.406233246205375e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 10.07, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.994310736656189, + "learning_rate": 9.83355361338397e-05, + "loss": 0.010001969523727894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01005, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.6052996516227722, + "learning_rate": 9.831590421230704e-05, + "loss": 0.002832446713000536, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00284, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.054735079407691956, + "learning_rate": 9.829615939924308e-05, + "loss": 0.0004141697136219591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00041, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.0013703533913940191, + "learning_rate": 9.827630174610754e-05, + "loss": 2.5566190743120387e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00003, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.31021150946617126, + "learning_rate": 9.825633130465418e-05, + "loss": 0.0015716712223365903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00157, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.0021009715273976326, + "learning_rate": 9.823624812693069e-05, + "loss": 2.7584441340877675e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.24123214185237885, + "learning_rate": 9.821605226527862e-05, + "loss": 0.001688711578026414, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00169, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.11157317459583282, + "learning_rate": 9.819574377233317e-05, + "loss": 0.000771663966588676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00077, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.0032444552052766085, + "learning_rate": 9.817532270102309e-05, + "loss": 3.7354184314608574e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00004, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.002511747879907489, + "learning_rate": 9.815478910457054e-05, + "loss": 3.364873191458173e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.20776121318340302, + "learning_rate": 9.813414303649097e-05, + "loss": 0.0014578676782548428, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00146, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 1.0314363241195679, + "learning_rate": 9.811338455059291e-05, + "loss": 0.007915408350527287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00795, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.011756581254303455, + "learning_rate": 9.809251370097791e-05, + "loss": 0.00010454728180775419, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.5820220112800598, + "learning_rate": 9.807153054204036e-05, + "loss": 0.012204312719404697, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01228, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 0.02615712210536003, + "learning_rate": 9.805043512846735e-05, + "loss": 0.0001216292439494282, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00012, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.0016786960186436772, + "learning_rate": 9.802922751523855e-05, + "loss": 2.662030965439044e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.6373363137245178, + "learning_rate": 9.800790775762605e-05, + "loss": 0.012163807637989521, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01224, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.022918498143553734, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0002818067150656134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.025542180985212326, + "learning_rate": 9.796493203179937e-05, + "loss": 0.00036140577867627144, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00036, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.0805443674325943, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0008338643237948418, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.010478734970092773, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0002059635880868882, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00021, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.07198218256235123, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0012385500594973564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00124, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.06538524478673935, + "learning_rate": 9.787763731193733e-05, + "loss": 0.001297011855058372, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.10911253094673157, + "learning_rate": 9.785553411578963e-05, + "loss": 0.0022439362946897745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00225, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.09725505858659744, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0012814714573323727, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00128, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.5701457262039185, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0053935484029352665, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00541, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 1.1999653577804565, + "learning_rate": 9.778855460904167e-05, + "loss": 0.011738242581486702, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01181, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.08431178331375122, + "learning_rate": 9.77660049946513e-05, + "loss": 0.0002786249096971005, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00028, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.2198057472705841, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0054946052841842175, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00551, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.15289317071437836, + "learning_rate": 9.77205714498407e-05, + "loss": 0.005137430503964424, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00515, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.08833356946706772, + "learning_rate": 9.769768763783107e-05, + "loss": 0.0010702629806473851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00107, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 1.050297737121582, + "learning_rate": 9.767469254550936e-05, + "loss": 0.01607959344983101, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01621, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 3.3124635219573975, + "learning_rate": 9.765158623280625e-05, + "loss": 0.05849085748195648, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06024, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.40984460711479187, + "learning_rate": 9.762836875994225e-05, + "loss": 0.00869081262499094, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00873, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.0552639476954937, + "learning_rate": 9.760504018742764e-05, + "loss": 0.0012600910849869251, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00126, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.25621575117111206, + "learning_rate": 9.75816005760622e-05, + "loss": 0.007940271869301796, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00797, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.1158919557929039, + "learning_rate": 9.755804998693513e-05, + "loss": 0.0022215445060282946, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00222, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.06458672136068344, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0011070814216509461, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00111, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.10265516489744186, + "learning_rate": 9.751061612119887e-05, + "loss": 0.0007613050402142107, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00076, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.5470341444015503, + "learning_rate": 9.748673296821358e-05, + "loss": 0.004083400592207909, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00409, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.023497454822063446, + "learning_rate": 9.746273908471418e-05, + "loss": 0.00018082497990690172, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00018, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.008276190608739853, + "learning_rate": 9.743863453323441e-05, + "loss": 5.542398139368743e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.001996487844735384, + "learning_rate": 9.741441937659645e-05, + "loss": 1.738916580507066e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.08108871430158615, + "learning_rate": 9.73900936779108e-05, + "loss": 0.0003538768214639276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00035, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.00038908072747290134, + "learning_rate": 9.736565750057593e-05, + "loss": 7.582004855066771e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.008329521864652634, + "learning_rate": 9.734111090827836e-05, + "loss": 3.9857310184743255e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.000670251261908561, + "learning_rate": 9.731645396499232e-05, + "loss": 6.995260719122598e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.0035178044345229864, + "learning_rate": 9.729168673497969e-05, + "loss": 1.869679181254469e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.2373104691505432, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0006409735069610178, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.013374611735343933, + "learning_rate": 9.724182167325899e-05, + "loss": 2.7810374376713298e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 0.0001627751043997705, + "learning_rate": 9.721672397151109e-05, + "loss": 1.8662318552742363e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.012986388057470322, + "learning_rate": 9.719151624295665e-05, + "loss": 3.3646811061771587e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00003, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.03348306566476822, + "learning_rate": 9.716619855329296e-05, + "loss": 6.960683822399005e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00007, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.00024256504548247904, + "learning_rate": 9.714077096850394e-05, + "loss": 9.166715813080373e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.3338123857975006, + "learning_rate": 9.711523355485992e-05, + "loss": 0.00018601410556584597, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00019, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 1.7860302250483073e-05, + "learning_rate": 9.708958637891746e-05, + "loss": 2.159395933176711e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 2.1244673916953616e-05, + "learning_rate": 9.706382950751918e-05, + "loss": 2.698998002870212e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 2.3123484425013885e-05, + "learning_rate": 9.703796300779361e-05, + "loss": 2.641835976646689e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 9.57277006818913e-06, + "learning_rate": 9.7011986947155e-05, + "loss": 1.2605066501691908e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.00038095240597613156, + "learning_rate": 9.698590139330309e-05, + "loss": 3.805786832344893e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 2.8026983737945557, + "learning_rate": 9.695970641422309e-05, + "loss": 0.04527268186211586, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04631, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 5.351627987693064e-05, + "learning_rate": 9.693340207818528e-05, + "loss": 6.533724672408425e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 3.1587586402893066, + "learning_rate": 9.690698845374505e-05, + "loss": 0.015918973833322525, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01605, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.14, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 1.1445614099502563, + "learning_rate": 9.688046560974256e-05, + "loss": 0.002670082962140441, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00267, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 1.6780225038528442, + "learning_rate": 9.685383361530262e-05, + "loss": 0.06125710532069206, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06317, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 1.3543906211853027, + "learning_rate": 9.682709253983455e-05, + "loss": 0.043765194714069366, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04474, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.16677653789520264, + "learning_rate": 9.680024245303193e-05, + "loss": 0.0009747418225742877, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.01408263947814703, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0004007547104265541, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0004, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.17440639436244965, + "learning_rate": 9.674621552561782e-05, + "loss": 0.004650459624826908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00466, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.3006541430950165, + "learning_rate": 9.671903882581331e-05, + "loss": 0.011185107752680779, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01125, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.11968870460987091, + "learning_rate": 9.669175339628791e-05, + "loss": 0.0045004296116530895, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00451, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.2765601873397827, + "learning_rate": 9.66643593081539e-05, + "loss": 0.011203736066818237, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01127, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.5569903254508972, + "learning_rate": 9.663685663280679e-05, + "loss": 0.006371541414409876, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00639, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.25860702991485596, + "learning_rate": 9.66092454419251e-05, + "loss": 0.007782225962728262, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00781, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 2.0391530990600586, + "learning_rate": 9.65815258074701e-05, + "loss": 0.010255386121571064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01031, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 8.181769371032715, + "learning_rate": 9.655369780168579e-05, + "loss": 0.06998364627361298, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.07249, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.08482059836387634, + "learning_rate": 9.652576149709852e-05, + "loss": 0.0010924283415079117, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00109, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 1.4669389724731445, + "learning_rate": 9.649771696651697e-05, + "loss": 0.026025421917438507, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02637, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.7967074513435364, + "learning_rate": 9.646956428303183e-05, + "loss": 0.05288422480225563, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05431, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.0539541058242321, + "learning_rate": 9.644130352001567e-05, + "loss": 0.000996746588498354, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.001, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.05697805434465408, + "learning_rate": 9.641293475112275e-05, + "loss": 0.0019096891628578305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00191, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.15788930654525757, + "learning_rate": 9.638445805028884e-05, + "loss": 0.0043489160016179085, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00436, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.475947767496109, + "learning_rate": 9.635587349173095e-05, + "loss": 0.012873997911810875, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01296, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 10.02, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.11763929575681686, + "learning_rate": 9.632718114994726e-05, + "loss": 0.005082407500594854, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0051, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.047499485313892365, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0020001258235424757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.002, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.053257688879966736, + "learning_rate": 9.626947341609934e-05, + "loss": 0.0016418047016486526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.19376324117183685, + "learning_rate": 9.624045817443519e-05, + "loss": 0.0031313879881054163, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00314, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.25925350189208984, + "learning_rate": 9.621133545034491e-05, + "loss": 0.0069334846921265125, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00696, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.5349475145339966, + "learning_rate": 9.61821053197293e-05, + "loss": 0.017092591151595116, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01724, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.006722787860780954, + "learning_rate": 9.615276785876898e-05, + "loss": 0.00013336099800653756, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00013, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.012635000981390476, + "learning_rate": 9.612332314392435e-05, + "loss": 0.0002897820377256721, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00029, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.03356361389160156, + "learning_rate": 9.609377125193535e-05, + "loss": 0.00026462774258106947, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00026, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.008992434479296207, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00014277567970566452, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 0.4329150319099426, + "learning_rate": 9.603434624488023e-05, + "loss": 0.013562595471739769, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01365, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.0025248134043067694, + "learning_rate": 9.600447328468982e-05, + "loss": 5.918296301388182e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.0018213920993730426, + "learning_rate": 9.597449345710597e-05, + "loss": 6.278142245719209e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.5386003851890564, + "learning_rate": 9.594440684026324e-05, + "loss": 0.005554460920393467, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00557, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.004698917269706726, + "learning_rate": 9.591421351257449e-05, + "loss": 0.00011057179654017091, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.005869440734386444, + "learning_rate": 9.588391355273066e-05, + "loss": 0.00012926939234603196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.7246658205986023, + "learning_rate": 9.585350703970069e-05, + "loss": 0.0022534995805472136, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00226, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 1.0683232545852661, + "learning_rate": 9.582299405273118e-05, + "loss": 0.006315313745290041, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00634, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.009510673582553864, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00016320317809004337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.01090356893837452, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0001996747450903058, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0002, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.08387676626443863, + "learning_rate": 9.573081704481236e-05, + "loss": 0.0007907926919870079, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00079, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.1897941380739212, + "learning_rate": 9.569987896009734e-05, + "loss": 0.0018501047743484378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.01003352366387844, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00016643876733724028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00017, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.25508517026901245, + "learning_rate": 9.563768465093079e-05, + "loss": 0.0015241497894749045, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00153, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.12474071979522705, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00047439540503546596, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00047, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.15887834131717682, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0012022990267723799, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0012, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.0015826806193217635, + "learning_rate": 9.554359905560886e-05, + "loss": 6.414140079868957e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00006, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.8478289246559143, + "learning_rate": 9.551202574875235e-05, + "loss": 0.002341005252674222, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00234, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.025573493912816048, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0001343936164630577, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.0026671665254980326, + "learning_rate": 9.544856246572735e-05, + "loss": 4.064952736371197e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.0014051123289391398, + "learning_rate": 9.541667265495922e-05, + "loss": 4.675618765759282e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.2168407291173935, + "learning_rate": 9.538467750874569e-05, + "loss": 0.03523726388812065, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03587, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.0012685427209362388, + "learning_rate": 9.535257711047364e-05, + "loss": 4.508057463681325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.8553150296211243, + "learning_rate": 9.532037154380439e-05, + "loss": 0.025239642709493637, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02556, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.1590186208486557, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0004924478125758469, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00049, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.01861356757581234, + "learning_rate": 9.525564524128947e-05, + "loss": 0.0003756298392545432, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00038, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.03878219425678253, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0009251241572201252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00093, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.06500300019979477, + "learning_rate": 9.519049927596896e-05, + "loss": 0.0017165483441203833, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00172, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.012984351254999638, + "learning_rate": 9.515776913181807e-05, + "loss": 0.00043050089152529836, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.9554085731506348, + "learning_rate": 9.512493432698582e-05, + "loss": 0.022500813007354736, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02276, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.06780962646007538, + "learning_rate": 9.509199494704748e-05, + "loss": 0.0015775745268911123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00158, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.12043783068656921, + "learning_rate": 9.505895107785091e-05, + "loss": 0.002761447336524725, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00277, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.2550680637359619, + "learning_rate": 9.502580280551627e-05, + "loss": 0.0021937121637165546, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0022, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.22257742285728455, + "learning_rate": 9.499255021643582e-05, + "loss": 0.005736821796745062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00575, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.8161686062812805, + "learning_rate": 9.495919339727372e-05, + "loss": 0.02995053492486477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0304, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.17400598526000977, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0006721575045958161, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00067, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.5554174184799194, + "learning_rate": 9.489216741671916e-05, + "loss": 0.02686231955885887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02723, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 3.851907253265381, + "learning_rate": 9.485849843001232e-05, + "loss": 0.014627727679908276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01474, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.46317151188850403, + "learning_rate": 9.48247255625946e-05, + "loss": 0.01176954060792923, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01184, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.05679527670145035, + "learning_rate": 9.479084890248611e-05, + "loss": 0.0013346781488507986, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00134, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.05617646127939224, + "learning_rate": 9.475686853797749e-05, + "loss": 0.0012955943820998073, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0013, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.12257706373929977, + "learning_rate": 9.472278455762962e-05, + "loss": 0.003295040922239423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0033, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.0822661742568016, + "learning_rate": 9.468859705027345e-05, + "loss": 0.002866032999008894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00287, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.1836707442998886, + "learning_rate": 9.465430610500977e-05, + "loss": 0.005489380564540625, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0055, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.23293602466583252, + "learning_rate": 9.461991181120889e-05, + "loss": 0.003381692338734865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00339, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.05416075140237808, + "learning_rate": 9.458541425851051e-05, + "loss": 0.0012558875605463982, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00126, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.08709446340799332, + "learning_rate": 9.455081353682347e-05, + "loss": 0.0021735774353146553, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00218, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.4669940173625946, + "learning_rate": 9.451610973632543e-05, + "loss": 0.014700866304337978, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01481, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.04177068918943405, + "learning_rate": 9.448130294746276e-05, + "loss": 0.0009484485490247607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00095, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.012790173292160034, + "learning_rate": 9.444639326095022e-05, + "loss": 0.0003626626275945455, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00036, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 1.1132054328918457, + "learning_rate": 9.441138076777071e-05, + "loss": 0.023097209632396698, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02337, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.007151913829147816, + "learning_rate": 9.437626555917512e-05, + "loss": 0.00016600379603914917, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.010779544711112976, + "learning_rate": 9.434104772668203e-05, + "loss": 0.00027127741486765444, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00027, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.1, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.002494187792763114, + "learning_rate": 9.430572736207746e-05, + "loss": 0.00010258900874760002, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.014571974985301495, + "learning_rate": 9.427030455741466e-05, + "loss": 0.0001752122916514054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00018, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005204187240451574, + "learning_rate": 9.423477940501391e-05, + "loss": 0.00011710889521054924, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0038825208321213722, + "learning_rate": 9.419915199746214e-05, + "loss": 9.639768541092053e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.17287366092205048, + "learning_rate": 9.416342242761287e-05, + "loss": 0.0006201150245033205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00062, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.002964016981422901, + "learning_rate": 9.412759078858583e-05, + "loss": 8.631067612441257e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00009, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.024093005806207657, + "learning_rate": 9.409165717376678e-05, + "loss": 0.0003696970525197685, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00037, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00637121731415391, + "learning_rate": 9.405562167680723e-05, + "loss": 0.00022675329819321632, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00023, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.013249519281089306, + "learning_rate": 9.401948439162429e-05, + "loss": 0.0001507245615357533, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00015, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.010370856150984764, + "learning_rate": 9.398324541240028e-05, + "loss": 0.00029673389508388937, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0003, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.005587432999163866, + "learning_rate": 9.394690483358256e-05, + "loss": 0.0001745278132148087, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.004225320182740688, + "learning_rate": 9.391046274988333e-05, + "loss": 0.00014175988326314837, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.0013623819686472416, + "learning_rate": 9.387391925627933e-05, + "loss": 3.905296034645289e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.24, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.005137237720191479, + "learning_rate": 9.383727444801156e-05, + "loss": 7.005511724855751e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.4223204553127289, + "learning_rate": 9.380052842058508e-05, + "loss": 0.013293527066707611, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01338, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0589374341070652, + "learning_rate": 9.376368126976879e-05, + "loss": 0.0003874894173350185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00039, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.22437813878059387, + "learning_rate": 9.37267330915951e-05, + "loss": 0.0021422698628157377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00214, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.017364539206027985, + "learning_rate": 9.368968398235973e-05, + "loss": 0.00038539565866813064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00039, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.69, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 0.02143651619553566, + "learning_rate": 9.365253403862147e-05, + "loss": 0.0003793695941567421, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00038, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.003489129710942507, + "learning_rate": 9.361528335720188e-05, + "loss": 9.892375965137035e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0017010540468618274, + "learning_rate": 9.35779320351851e-05, + "loss": 5.5953427363419905e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.11703313142061234, + "learning_rate": 9.354048016991751e-05, + "loss": 0.0012725440319627523, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00127, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.004782855045050383, + "learning_rate": 9.35029278590076e-05, + "loss": 0.00011988157348241657, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00012, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.005790780298411846, + "learning_rate": 9.346527520032561e-05, + "loss": 0.00011395225737942383, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.0037311515770852566, + "learning_rate": 9.342752229200325e-05, + "loss": 0.0001015245943563059, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 0.02230995148420334, + "learning_rate": 9.338966923243363e-05, + "loss": 0.00022116347099654377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00022, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.12711703777313232, + "learning_rate": 9.335171612027078e-05, + "loss": 0.0006741612451151013, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00067, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 0.35434964299201965, + "learning_rate": 9.331366305442954e-05, + "loss": 0.0009926246711984277, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00099, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.002094523748382926, + "learning_rate": 9.32755101340852e-05, + "loss": 5.7496239605825394e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00006, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.0017241680761799216, + "learning_rate": 9.323725745867336e-05, + "loss": 5.681024049408734e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.4935676157474518, + "learning_rate": 9.319890512788957e-05, + "loss": 0.006042602472007275, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00606, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.0015609359834343195, + "learning_rate": 9.316045324168912e-05, + "loss": 4.8061745474115014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.33837297558784485, + "learning_rate": 9.312190190028672e-05, + "loss": 0.01984686404466629, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02005, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.001104304799810052, + "learning_rate": 9.308325120415636e-05, + "loss": 2.6898589567281306e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 0.0009272405877709389, + "learning_rate": 9.30445012540309e-05, + "loss": 2.8348587875370868e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 0.00263971253298223, + "learning_rate": 9.300565215090192e-05, + "loss": 6.861943984404206e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.24433688819408417, + "learning_rate": 9.296670399601942e-05, + "loss": 0.0012602279894053936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00126, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 0.00669356482103467, + "learning_rate": 9.292765689089152e-05, + "loss": 0.00013748394849244505, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 0.008884169161319733, + "learning_rate": 9.288851093728427e-05, + "loss": 0.00020161936117801815, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.03253387659788132, + "learning_rate": 9.284926623722131e-05, + "loss": 0.0007488323608413339, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 0.48039138317108154, + "learning_rate": 9.280992289298363e-05, + "loss": 0.013669916428625584, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01376, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.04280317574739456, + "learning_rate": 9.277048100710934e-05, + "loss": 0.000626624736469239, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00063, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 0.009100711904466152, + "learning_rate": 9.273094068239336e-05, + "loss": 0.0002161133015761152, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00022, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.6721935868263245, + "learning_rate": 9.269130202188718e-05, + "loss": 0.003257373347878456, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00326, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.006734038703143597, + "learning_rate": 9.265156512889854e-05, + "loss": 0.00016828924708534032, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 0.016732869669795036, + "learning_rate": 9.261173010699122e-05, + "loss": 0.00036356697091832757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00036, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.003918794449418783, + "learning_rate": 9.257179705998475e-05, + "loss": 0.00011146099132020026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 0.006524030584841967, + "learning_rate": 9.253176609195413e-05, + "loss": 0.00015829743642825633, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 0.4059728980064392, + "learning_rate": 9.249163730722958e-05, + "loss": 0.001720082014799118, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00172, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.006374079268425703, + "learning_rate": 9.245141081039623e-05, + "loss": 0.00010285466851200908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 0.04326994717121124, + "learning_rate": 9.241108670629387e-05, + "loss": 0.0006445100298151374, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.013943824917078018, + "learning_rate": 9.23706651000167e-05, + "loss": 0.00018365212599746883, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00018, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.002007151721045375, + "learning_rate": 9.233014609691304e-05, + "loss": 4.110952068003826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.0017673998372629285, + "learning_rate": 9.2289529802585e-05, + "loss": 5.1574999815784395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.004930772352963686, + "learning_rate": 9.224881632288832e-05, + "loss": 0.0001214695003000088, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.0032258464489132166, + "learning_rate": 9.2208005763932e-05, + "loss": 7.19371746527031e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00007, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.0031119745690375566, + "learning_rate": 9.2167098232078e-05, + "loss": 4.87166726088617e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 0.0016869365936145186, + "learning_rate": 9.21260938339411e-05, + "loss": 2.954686715384014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.0038998830132186413, + "learning_rate": 9.208499267638847e-05, + "loss": 7.866357918828726e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 0.003540599253028631, + "learning_rate": 9.204379486653952e-05, + "loss": 4.6444743929896504e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00005, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 0.0027000850532203913, + "learning_rate": 9.20025005117655e-05, + "loss": 6.059448060113937e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00006, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 0.00047282822197303176, + "learning_rate": 9.19611097196893e-05, + "loss": 1.3409662642516196e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 0.0004859396431129426, + "learning_rate": 9.191962259818518e-05, + "loss": 1.4546509191859514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.0019297238904982805, + "learning_rate": 9.18780392553784e-05, + "loss": 1.886364771053195e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.006912007462233305, + "learning_rate": 9.183635979964503e-05, + "loss": 4.965384869137779e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00005, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.12616240978240967, + "learning_rate": 9.179458433961166e-05, + "loss": 0.0001120424349210225, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00011, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.0003369607147760689, + "learning_rate": 9.1752712984155e-05, + "loss": 9.538351150695235e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.0004897752078250051, + "learning_rate": 9.17107458424018e-05, + "loss": 9.365672667627223e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.001080889138393104, + "learning_rate": 9.166868302372835e-05, + "loss": 1.9623625121312216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.001739430008456111, + "learning_rate": 9.162652463776038e-05, + "loss": 2.61274090007646e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.00015428484766744077, + "learning_rate": 9.158427079437264e-05, + "loss": 5.0580983952386305e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.0011717748129740357, + "learning_rate": 9.154192160368866e-05, + "loss": 2.2226362489163876e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.00020704242342617363, + "learning_rate": 9.149947717608052e-05, + "loss": 4.180368250672473e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.0032859130296856165, + "learning_rate": 9.145693762216845e-05, + "loss": 5.2759838581550866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00005, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.0019501899369060993, + "learning_rate": 9.141430305282065e-05, + "loss": 9.430449608771596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 2.7989144325256348, + "learning_rate": 9.137157357915291e-05, + "loss": 0.013717958703637123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01381, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 0.00013671681517735124, + "learning_rate": 9.132874931252841e-05, + "loss": 4.223028554406483e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.00048552214866504073, + "learning_rate": 9.128583036455732e-05, + "loss": 6.883431979076704e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.00017881239182315767, + "learning_rate": 9.124281684709663e-05, + "loss": 5.3810631470696535e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.0002475245564710349, + "learning_rate": 9.119970887224978e-05, + "loss": 7.255342097778339e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.00019574622274376452, + "learning_rate": 9.11565065523664e-05, + "loss": 5.540817710425472e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.00034243447589688003, + "learning_rate": 9.111321000004195e-05, + "loss": 6.973566087253857e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.003270970657467842, + "learning_rate": 9.106981932811758e-05, + "loss": 2.941479215223808e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.016538448631763458, + "learning_rate": 9.102633464967965e-05, + "loss": 9.504774789093062e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.08727161586284637, + "learning_rate": 9.098275607805957e-05, + "loss": 0.0004140432574786246, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00041, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 2.2421369552612305, + "learning_rate": 9.093908372683343e-05, + "loss": 0.03778925538063049, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03851, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.46441370248794556, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0029845235403627157, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00299, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0004926464753225446, + "learning_rate": 9.085145814108921e-05, + "loss": 1.2092392353224568e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0015016683610156178, + "learning_rate": 9.08075051349442e-05, + "loss": 3.592526991269551e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.3595995306968689, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0065900967456400394, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00661, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.00954297836869955, + "learning_rate": 9.071931926886792e-05, + "loss": 0.00021393387578427792, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00021, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.5538262128829956, + "learning_rate": 9.067508663876997e-05, + "loss": 0.004892576020210981, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0049, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.05068283528089523, + "learning_rate": 9.063076103092557e-05, + "loss": 0.0015409812331199646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00154, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 1.623219609260559, + "learning_rate": 9.05863425608578e-05, + "loss": 0.016946446150541306, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01709, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.30548763275146484, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002110418863594532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00211, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 165520 + }, + { + "epoch": 8.015625, + "grad_norm": 0.03473364934325218, + "learning_rate": 9.049722749735415e-05, + "loss": 0.00032757490407675505, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00033, + "step": 513, + "tokens/total": 7745176, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 165842 + }, + { + "epoch": 8.03125, + "grad_norm": 0.053817443549633026, + "learning_rate": 9.04525311361733e-05, + "loss": 0.0004164045094512403, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00042, + "step": 514, + "tokens/total": 7760316, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 166164 + }, + { + "epoch": 8.046875, + "grad_norm": 2.463132381439209, + "learning_rate": 9.04077423772785e-05, + "loss": 0.007824876345694065, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00786, + "step": 515, + "tokens/total": 7775516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 166490 + }, + { + "epoch": 8.0625, + "grad_norm": 0.6209765672683716, + "learning_rate": 9.036286133739987e-05, + "loss": 0.011545822024345398, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01161, + "step": 516, + "tokens/total": 7790668, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 166814 + }, + { + "epoch": 8.078125, + "grad_norm": 0.29986634850502014, + "learning_rate": 9.031788813350807e-05, + "loss": 0.01826884225010872, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01844, + "step": 517, + "tokens/total": 7805712, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 167137 + }, + { + "epoch": 8.09375, + "grad_norm": 0.061234600841999054, + "learning_rate": 9.027282288281396e-05, + "loss": 0.0011736912420019507, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00117, + "step": 518, + "tokens/total": 7820848, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 167461 + }, + { + "epoch": 8.109375, + "grad_norm": 0.34970536828041077, + "learning_rate": 9.022766570276829e-05, + "loss": 0.0028101038187742233, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00281, + "step": 519, + "tokens/total": 7835976, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 167784 + }, + { + "epoch": 8.125, + "grad_norm": 0.037389740347862244, + "learning_rate": 9.018241671106134e-05, + "loss": 0.0008111234055832028, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00081, + "step": 520, + "tokens/total": 7851168, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 168106 + }, + { + "epoch": 8.140625, + "grad_norm": 0.025489794090390205, + "learning_rate": 9.013707602562279e-05, + "loss": 0.0008077127858996391, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00081, + "step": 521, + "tokens/total": 7866244, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 168434 + }, + { + "epoch": 8.15625, + "grad_norm": 0.014333931729197502, + "learning_rate": 9.009164376462119e-05, + "loss": 0.0002727667160797864, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00027, + "step": 522, + "tokens/total": 7881132, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 168757 + }, + { + "epoch": 8.171875, + "grad_norm": 0.04828103631734848, + "learning_rate": 9.004612004646383e-05, + "loss": 0.0011419751681387424, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00114, + "step": 523, + "tokens/total": 7896264, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 169077 + }, + { + "epoch": 8.1875, + "grad_norm": 0.018072057515382767, + "learning_rate": 9.00005049897963e-05, + "loss": 0.0005876998184248805, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00059, + "step": 524, + "tokens/total": 7911500, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 169403 + }, + { + "epoch": 8.203125, + "grad_norm": 0.10106753557920456, + "learning_rate": 8.995479871350229e-05, + "loss": 0.0016654415521770716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00167, + "step": 525, + "tokens/total": 7926620, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 169726 + }, + { + "epoch": 8.21875, + "grad_norm": 0.023626673966646194, + "learning_rate": 8.990900133670319e-05, + "loss": 0.0007265466265380383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00073, + "step": 526, + "tokens/total": 7941640, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 170048 + }, + { + "epoch": 8.234375, + "grad_norm": 0.047687333077192307, + "learning_rate": 8.986311297875786e-05, + "loss": 0.0006352646159939468, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00064, + "step": 527, + "tokens/total": 7956708, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 170366 + }, + { + "epoch": 8.25, + "grad_norm": 0.011076240800321102, + "learning_rate": 8.981713375926225e-05, + "loss": 0.0003708331205416471, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 528, + "tokens/total": 7971900, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 170688 + }, + { + "epoch": 8.265625, + "grad_norm": 0.3267386555671692, + "learning_rate": 8.977106379804911e-05, + "loss": 0.002543328097090125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00255, + "step": 529, + "tokens/total": 7986976, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 171014 + }, + { + "epoch": 8.28125, + "grad_norm": 0.043074894696474075, + "learning_rate": 8.97249032151877e-05, + "loss": 0.0005432984326034784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 530, + "tokens/total": 8001928, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 171337 + }, + { + "epoch": 8.296875, + "grad_norm": 0.3724943697452545, + "learning_rate": 8.967865213098347e-05, + "loss": 0.0028712605126202106, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00288, + "step": 531, + "tokens/total": 8017012, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 171662 + }, + { + "epoch": 8.3125, + "grad_norm": 0.1613280475139618, + "learning_rate": 8.963231066597771e-05, + "loss": 0.0010274314554408193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00103, + "step": 532, + "tokens/total": 8032012, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 171987 + }, + { + "epoch": 8.328125, + "grad_norm": 0.0017801767680794, + "learning_rate": 8.95858789409473e-05, + "loss": 4.680546044255607e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 533, + "tokens/total": 8047000, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 172313 + }, + { + "epoch": 8.34375, + "grad_norm": 0.0016095931641757488, + "learning_rate": 8.953935707690434e-05, + "loss": 4.314066609367728e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00004, + "step": 534, + "tokens/total": 8062088, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 172637 + }, + { + "epoch": 8.359375, + "grad_norm": 0.01733282208442688, + "learning_rate": 8.949274519509585e-05, + "loss": 0.00013291885261423886, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 535, + "tokens/total": 8077132, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 172961 + }, + { + "epoch": 8.375, + "grad_norm": 0.0012273149332031608, + "learning_rate": 8.944604341700346e-05, + "loss": 3.25683577102609e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 536, + "tokens/total": 8092276, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 173283 + }, + { + "epoch": 8.390625, + "grad_norm": 0.00094650435494259, + "learning_rate": 8.939925186434308e-05, + "loss": 4.1214312659576535e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 537, + "tokens/total": 8107396, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 173603 + }, + { + "epoch": 8.40625, + "grad_norm": 0.0023045814596116543, + "learning_rate": 8.93523706590646e-05, + "loss": 2.960239908134099e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 538, + "tokens/total": 8122408, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 173926 + }, + { + "epoch": 8.421875, + "grad_norm": 0.0005694059655070305, + "learning_rate": 8.930539992335164e-05, + "loss": 1.7956097508431412e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 539, + "tokens/total": 8137452, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 174247 + }, + { + "epoch": 8.4375, + "grad_norm": 0.0006300202803686261, + "learning_rate": 8.9258339779621e-05, + "loss": 1.8808523236657493e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 540, + "tokens/total": 8152540, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 174573 + }, + { + "epoch": 8.453125, + "grad_norm": 0.0008573881350457668, + "learning_rate": 8.921119035052265e-05, + "loss": 2.2247711967793293e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 541, + "tokens/total": 8167692, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 174898 + }, + { + "epoch": 8.46875, + "grad_norm": 0.0002639936574269086, + "learning_rate": 8.916395175893916e-05, + "loss": 1.0038807886303402e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 542, + "tokens/total": 8182760, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 175225 + }, + { + "epoch": 8.484375, + "grad_norm": 0.00023128512839321047, + "learning_rate": 8.911662412798553e-05, + "loss": 8.707731467438862e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 543, + "tokens/total": 8197968, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 175550 + }, + { + "epoch": 8.5, + "grad_norm": 0.00045672088162973523, + "learning_rate": 8.906920758100882e-05, + "loss": 1.7010246665449813e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 544, + "tokens/total": 8213064, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 175872 + }, + { + "epoch": 8.515625, + "grad_norm": 0.00044114937190897763, + "learning_rate": 8.902170224158778e-05, + "loss": 1.0676423698896542e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 545, + "tokens/total": 8228148, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 176193 + }, + { + "epoch": 8.53125, + "grad_norm": 0.00028124169330112636, + "learning_rate": 8.897410823353264e-05, + "loss": 1.1783889021899085e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 546, + "tokens/total": 8243252, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 176514 + }, + { + "epoch": 8.546875, + "grad_norm": 0.0003170839336235076, + "learning_rate": 8.892642568088465e-05, + "loss": 8.978167898021638e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 547, + "tokens/total": 8258448, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 176839 + }, + { + "epoch": 8.5625, + "grad_norm": 0.000917276251129806, + "learning_rate": 8.887865470791589e-05, + "loss": 1.465234709030483e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 548, + "tokens/total": 8273528, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 177167 + }, + { + "epoch": 8.578125, + "grad_norm": 0.0006104891654103994, + "learning_rate": 8.883079543912883e-05, + "loss": 1.531057569081895e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 549, + "tokens/total": 8288496, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 177491 + }, + { + "epoch": 8.59375, + "grad_norm": 0.0004606851434800774, + "learning_rate": 8.878284799925612e-05, + "loss": 1.0263601325277705e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00001, + "step": 550, + "tokens/total": 8303696, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 177818 + }, + { + "epoch": 8.609375, + "grad_norm": 0.004411522299051285, + "learning_rate": 8.873481251326016e-05, + "loss": 4.7359702875837684e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 551, + "tokens/total": 8318700, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 178144 + }, + { + "epoch": 8.625, + "grad_norm": 0.0004904805682599545, + "learning_rate": 8.868668910633283e-05, + "loss": 8.810208782961126e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 552, + "tokens/total": 8333792, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 178466 + }, + { + "epoch": 8.640625, + "grad_norm": 0.0002976775576826185, + "learning_rate": 8.863847790389513e-05, + "loss": 8.980839083960745e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 553, + "tokens/total": 8348860, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 178793 + }, + { + "epoch": 8.65625, + "grad_norm": 0.5571802854537964, + "learning_rate": 8.859017903159695e-05, + "loss": 0.04063796252012253, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04147, + "step": 554, + "tokens/total": 8364084, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 179113 + }, + { + "epoch": 8.671875, + "grad_norm": 0.005246421322226524, + "learning_rate": 8.854179261531659e-05, + "loss": 3.459662912064232e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00003, + "step": 555, + "tokens/total": 8379180, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 179436 + }, + { + "epoch": 8.6875, + "grad_norm": 0.0004945784457959235, + "learning_rate": 8.849331878116055e-05, + "loss": 1.6462972780573182e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 556, + "tokens/total": 8394380, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 179753 + }, + { + "epoch": 8.703125, + "grad_norm": 0.00044819965842179954, + "learning_rate": 8.844475765546314e-05, + "loss": 2.100240089930594e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00002, + "step": 557, + "tokens/total": 8409332, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 180074 + }, + { + "epoch": 8.71875, + "grad_norm": 0.0010908502154052258, + "learning_rate": 8.83961093647862e-05, + "loss": 3.675068001030013e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00004, + "step": 558, + "tokens/total": 8424388, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 180403 + }, + { + "epoch": 8.734375, + "grad_norm": 0.0012147148372605443, + "learning_rate": 8.834737403591869e-05, + "loss": 4.8132918891496956e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 559, + "tokens/total": 8439436, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 180730 + }, + { + "epoch": 8.75, + "grad_norm": 0.2376457154750824, + "learning_rate": 8.82985517958765e-05, + "loss": 0.0002953725343104452, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0003, + "step": 560, + "tokens/total": 8454568, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 181051 + }, + { + "epoch": 8.765625, + "grad_norm": 0.0030292074661701918, + "learning_rate": 8.824964277190194e-05, + "loss": 9.119488822761923e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00009, + "step": 561, + "tokens/total": 8469748, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 181374 + }, + { + "epoch": 8.78125, + "grad_norm": 0.004703894257545471, + "learning_rate": 8.820064709146357e-05, + "loss": 0.00017599575221538544, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00018, + "step": 562, + "tokens/total": 8484772, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 181698 + }, + { + "epoch": 8.796875, + "grad_norm": 0.004817877430468798, + "learning_rate": 8.815156488225575e-05, + "loss": 0.0001848819520091638, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00018, + "step": 563, + "tokens/total": 8499748, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 182020 + }, + { + "epoch": 8.8125, + "grad_norm": 0.0038356403820216656, + "learning_rate": 8.810239627219837e-05, + "loss": 0.00017618865240365267, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00018, + "step": 564, + "tokens/total": 8514840, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 182344 + }, + { + "epoch": 8.828125, + "grad_norm": 0.004524700343608856, + "learning_rate": 8.805314138943649e-05, + "loss": 0.00019838388834614307, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0002, + "step": 565, + "tokens/total": 8529904, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 182664 + }, + { + "epoch": 8.84375, + "grad_norm": 0.007456054911017418, + "learning_rate": 8.800380036234005e-05, + "loss": 0.00023415160831063986, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 566, + "tokens/total": 8545076, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 182984 + }, + { + "epoch": 8.859375, + "grad_norm": 0.006181093864142895, + "learning_rate": 8.795437331950349e-05, + "loss": 0.00023403027444146574, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00023, + "step": 567, + "tokens/total": 8560164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 183306 + }, + { + "epoch": 8.875, + "grad_norm": 0.004892744589596987, + "learning_rate": 8.790486038974539e-05, + "loss": 0.00016624186537228525, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00017, + "step": 568, + "tokens/total": 8575364, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 183630 + }, + { + "epoch": 8.890625, + "grad_norm": 0.03981661796569824, + "learning_rate": 8.785526170210823e-05, + "loss": 0.0012382965069264174, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00124, + "step": 569, + "tokens/total": 8590504, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 183950 + }, + { + "epoch": 8.90625, + "grad_norm": 0.0056670899502933025, + "learning_rate": 8.780557738585795e-05, + "loss": 0.00020142635912634432, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0002, + "step": 570, + "tokens/total": 8605536, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 184273 + }, + { + "epoch": 8.921875, + "grad_norm": 0.006030660588294268, + "learning_rate": 8.775580757048369e-05, + "loss": 0.0002046318695647642, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0002, + "step": 571, + "tokens/total": 8620496, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 184593 + }, + { + "epoch": 8.9375, + "grad_norm": 0.03583228588104248, + "learning_rate": 8.770595238569737e-05, + "loss": 0.00024410657351836562, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00024, + "step": 572, + "tokens/total": 8635532, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 184917 + }, + { + "epoch": 8.953125, + "grad_norm": 0.005166627932339907, + "learning_rate": 8.765601196143347e-05, + "loss": 0.00015040149446576834, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00015, + "step": 573, + "tokens/total": 8650684, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 185241 + }, + { + "epoch": 8.96875, + "grad_norm": 0.005641022231429815, + "learning_rate": 8.76059864278486e-05, + "loss": 0.00023396796314045787, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00023, + "step": 574, + "tokens/total": 8665752, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 185565 + }, + { + "epoch": 8.984375, + "grad_norm": 0.010190443135797977, + "learning_rate": 8.755587591532116e-05, + "loss": 0.00030070863431319594, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0003, + "step": 575, + "tokens/total": 8680856, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 185886 + }, + { + "epoch": 9.0, + "grad_norm": 0.002653313335031271, + "learning_rate": 8.750568055445105e-05, + "loss": 0.00011337960313539952, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00011, + "step": 576, + "tokens/total": 8695996, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 186210 + }, + { + "epoch": 9.015625, + "grad_norm": 0.004400210455060005, + "learning_rate": 8.74554004760593e-05, + "loss": 8.902224362827837e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00009, + "step": 577, + "tokens/total": 8711112, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 186533 + }, + { + "epoch": 9.03125, + "grad_norm": 0.001120831468142569, + "learning_rate": 8.740503581118774e-05, + "loss": 5.5197382607730106e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 578, + "tokens/total": 8726132, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 186855 + }, + { + "epoch": 9.046875, + "grad_norm": 0.0029407162219285965, + "learning_rate": 8.735458669109863e-05, + "loss": 9.355632937513292e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00009, + "step": 579, + "tokens/total": 8741288, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 187179 + }, + { + "epoch": 9.0625, + "grad_norm": 0.0049383873119950294, + "learning_rate": 8.730405324727435e-05, + "loss": 0.00012397070531733334, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00012, + "step": 580, + "tokens/total": 8756480, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 187503 + }, + { + "epoch": 9.078125, + "grad_norm": 0.0018089935183525085, + "learning_rate": 8.72534356114171e-05, + "loss": 7.31149993953295e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00007, + "step": 581, + "tokens/total": 8771664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 187824 + }, + { + "epoch": 9.09375, + "grad_norm": 0.00066189473727718, + "learning_rate": 8.720273391544839e-05, + "loss": 3.5545119317248464e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 582, + "tokens/total": 8786788, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 188147 + }, + { + "epoch": 9.109375, + "grad_norm": 0.005104087758809328, + "learning_rate": 8.715194829150892e-05, + "loss": 0.00015624079969711602, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00016, + "step": 583, + "tokens/total": 8801964, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 188471 + }, + { + "epoch": 9.125, + "grad_norm": 0.0004942028899677098, + "learning_rate": 8.710107887195807e-05, + "loss": 2.6367362806922756e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 584, + "tokens/total": 8817248, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 188799 + }, + { + "epoch": 9.140625, + "grad_norm": 0.0009785937145352364, + "learning_rate": 8.705012578937364e-05, + "loss": 4.363520929473452e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 585, + "tokens/total": 8832416, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 189119 + }, + { + "epoch": 9.15625, + "grad_norm": 0.0011245476780459285, + "learning_rate": 8.699908917655145e-05, + "loss": 4.6138473408063874e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00005, + "step": 586, + "tokens/total": 8847412, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 189437 + }, + { + "epoch": 9.171875, + "grad_norm": 0.0005509463371708989, + "learning_rate": 8.694796916650503e-05, + "loss": 2.687949381652288e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 587, + "tokens/total": 8862508, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 189761 + }, + { + "epoch": 9.1875, + "grad_norm": 0.0005343457451090217, + "learning_rate": 8.689676589246528e-05, + "loss": 2.6335586881032214e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 588, + "tokens/total": 8877616, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 190085 + }, + { + "epoch": 9.203125, + "grad_norm": 0.0004939015489071608, + "learning_rate": 8.684547948788008e-05, + "loss": 2.5435567295062356e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 589, + "tokens/total": 8892752, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 190410 + }, + { + "epoch": 9.21875, + "grad_norm": 0.0005137353437021375, + "learning_rate": 8.679411008641397e-05, + "loss": 2.3000951841822825e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 590, + "tokens/total": 8907852, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 190732 + }, + { + "epoch": 9.234375, + "grad_norm": 0.0005416902713477612, + "learning_rate": 8.674265782194781e-05, + "loss": 2.4090912120300345e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 591, + "tokens/total": 8922884, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 191054 + }, + { + "epoch": 9.25, + "grad_norm": 0.00036916331737302244, + "learning_rate": 8.66911228285784e-05, + "loss": 1.89321581274271e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 592, + "tokens/total": 8937984, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 191383 + }, + { + "epoch": 9.265625, + "grad_norm": 0.0006170656997710466, + "learning_rate": 8.663950524061822e-05, + "loss": 2.661717553564813e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 593, + "tokens/total": 8953168, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 191709 + }, + { + "epoch": 9.28125, + "grad_norm": 0.0005721712368540466, + "learning_rate": 8.658780519259493e-05, + "loss": 2.5814701075432822e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 594, + "tokens/total": 8968100, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 192030 + }, + { + "epoch": 9.296875, + "grad_norm": 0.0003650241415016353, + "learning_rate": 8.653602281925113e-05, + "loss": 1.7348647816106677e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 595, + "tokens/total": 8983324, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 192352 + }, + { + "epoch": 9.3125, + "grad_norm": 0.0010323122842237353, + "learning_rate": 8.648415825554399e-05, + "loss": 2.5114946765825152e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 596, + "tokens/total": 8998508, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 192675 + }, + { + "epoch": 9.328125, + "grad_norm": 0.10986047238111496, + "learning_rate": 8.643221163664483e-05, + "loss": 0.00017984467558562756, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00018, + "step": 597, + "tokens/total": 9013568, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 192995 + }, + { + "epoch": 9.34375, + "grad_norm": 0.002174603519961238, + "learning_rate": 8.638018309793893e-05, + "loss": 4.609280586009845e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00005, + "step": 598, + "tokens/total": 9028760, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 193317 + }, + { + "epoch": 9.359375, + "grad_norm": 0.0004154450143687427, + "learning_rate": 8.632807277502501e-05, + "loss": 1.8876007743529044e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 599, + "tokens/total": 9043940, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 193644 + }, + { + "epoch": 9.375, + "grad_norm": 0.00039323154487647116, + "learning_rate": 8.627588080371493e-05, + "loss": 1.7299762475886382e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 600, + "tokens/total": 9059108, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 193969 + }, + { + "epoch": 9.390625, + "grad_norm": 0.0002693760325200856, + "learning_rate": 8.622360732003337e-05, + "loss": 1.28796946228249e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 601, + "tokens/total": 9074212, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 194291 + }, + { + "epoch": 9.40625, + "grad_norm": 0.00024431548081338406, + "learning_rate": 8.617125246021742e-05, + "loss": 1.2649304153455887e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 602, + "tokens/total": 9089180, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 194612 + }, + { + "epoch": 9.421875, + "grad_norm": 0.00026792712742462754, + "learning_rate": 8.611881636071627e-05, + "loss": 1.3494095583155286e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 603, + "tokens/total": 9104272, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 194933 + }, + { + "epoch": 9.4375, + "grad_norm": 0.0003894483670592308, + "learning_rate": 8.60662991581909e-05, + "loss": 1.0190186003455892e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 604, + "tokens/total": 9119376, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 195252 + }, + { + "epoch": 9.453125, + "grad_norm": 0.0006637090700678527, + "learning_rate": 8.601370098951355e-05, + "loss": 2.4055283574853092e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 605, + "tokens/total": 9134396, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 195574 + }, + { + "epoch": 9.46875, + "grad_norm": 0.001436291029676795, + "learning_rate": 8.596102199176755e-05, + "loss": 1.810987487260718e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 606, + "tokens/total": 9149608, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 195900 + }, + { + "epoch": 9.484375, + "grad_norm": 0.00026127498131245375, + "learning_rate": 8.59082623022469e-05, + "loss": 1.3558158570958767e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 607, + "tokens/total": 9164832, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 196225 + }, + { + "epoch": 9.5, + "grad_norm": 0.000587965885642916, + "learning_rate": 8.585542205845588e-05, + "loss": 1.37108927447116e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 608, + "tokens/total": 9179884, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 196550 + }, + { + "epoch": 9.515625, + "grad_norm": 0.0002404104743618518, + "learning_rate": 8.580250139810868e-05, + "loss": 1.0100266990775708e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 609, + "tokens/total": 9194960, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 196874 + }, + { + "epoch": 9.53125, + "grad_norm": 0.0008569957572035491, + "learning_rate": 8.57495004591291e-05, + "loss": 2.3998738470254466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 610, + "tokens/total": 9210108, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 197195 + }, + { + "epoch": 9.546875, + "grad_norm": 0.000233593862503767, + "learning_rate": 8.56964193796502e-05, + "loss": 1.0664000910765026e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 611, + "tokens/total": 9225252, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 197519 + }, + { + "epoch": 9.5625, + "grad_norm": 0.00019542351947166026, + "learning_rate": 8.56432582980139e-05, + "loss": 9.732106263982132e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 612, + "tokens/total": 9240320, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 197843 + }, + { + "epoch": 9.578125, + "grad_norm": 2.2527358531951904, + "learning_rate": 8.559001735277055e-05, + "loss": 0.00471502123400569, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00473, + "step": 613, + "tokens/total": 9255456, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 198165 + }, + { + "epoch": 9.59375, + "grad_norm": 0.00020894742920063436, + "learning_rate": 8.553669668267871e-05, + "loss": 1.0032665159087628e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 614, + "tokens/total": 9270556, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 198486 + }, + { + "epoch": 9.609375, + "grad_norm": 0.0005773341399617493, + "learning_rate": 8.548329642670472e-05, + "loss": 1.974812948901672e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 615, + "tokens/total": 9285660, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 198814 + }, + { + "epoch": 9.625, + "grad_norm": 0.00020035076886415482, + "learning_rate": 8.542981672402233e-05, + "loss": 9.573127499606926e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 616, + "tokens/total": 9300668, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 199136 + }, + { + "epoch": 9.640625, + "grad_norm": 0.00023058710212353617, + "learning_rate": 8.537625771401231e-05, + "loss": 1.0730314897955395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 617, + "tokens/total": 9315668, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 199460 + }, + { + "epoch": 9.65625, + "grad_norm": 0.00024055950052570552, + "learning_rate": 8.53226195362622e-05, + "loss": 1.1680522220558487e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 618, + "tokens/total": 9330848, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 199788 + }, + { + "epoch": 9.671875, + "grad_norm": 0.0003391138161532581, + "learning_rate": 8.526890233056579e-05, + "loss": 1.1571202776394784e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 619, + "tokens/total": 9345924, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 200110 + }, + { + "epoch": 9.6875, + "grad_norm": 0.003901060437783599, + "learning_rate": 8.521510623692288e-05, + "loss": 3.571059278328903e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 620, + "tokens/total": 9361068, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 200438 + }, + { + "epoch": 9.703125, + "grad_norm": 0.4270281493663788, + "learning_rate": 8.516123139553889e-05, + "loss": 0.04005718603730202, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.04087, + "step": 621, + "tokens/total": 9376068, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 200762 + }, + { + "epoch": 9.71875, + "grad_norm": 0.0002039646205957979, + "learning_rate": 8.510727794682443e-05, + "loss": 9.880464858724736e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 622, + "tokens/total": 9391068, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 201087 + }, + { + "epoch": 9.734375, + "grad_norm": 0.0005474572535604239, + "learning_rate": 8.505324603139499e-05, + "loss": 2.0959005269105546e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 623, + "tokens/total": 9406204, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 201407 + }, + { + "epoch": 9.75, + "grad_norm": 0.0035034625325351954, + "learning_rate": 8.499913579007058e-05, + "loss": 7.45462384657003e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00007, + "step": 624, + "tokens/total": 9421308, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 201727 + }, + { + "epoch": 9.765625, + "grad_norm": 0.0004683059232775122, + "learning_rate": 8.494494736387535e-05, + "loss": 2.068639878416434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00002, + "step": 625, + "tokens/total": 9436284, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 202052 + }, + { + "epoch": 9.78125, + "grad_norm": 0.09564242511987686, + "learning_rate": 8.489068089403719e-05, + "loss": 0.0005714035360142589, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00057, + "step": 626, + "tokens/total": 9451392, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 202375 + }, + { + "epoch": 9.796875, + "grad_norm": 0.008086667396128178, + "learning_rate": 8.483633652198739e-05, + "loss": 0.00019731087377294898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 627, + "tokens/total": 9466440, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 202706 + }, + { + "epoch": 9.8125, + "grad_norm": 0.001999929780140519, + "learning_rate": 8.47819143893603e-05, + "loss": 7.604056008858606e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 628, + "tokens/total": 9481532, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 203028 + }, + { + "epoch": 9.828125, + "grad_norm": 0.0049367668107151985, + "learning_rate": 8.472741463799296e-05, + "loss": 0.00014261776232160628, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 629, + "tokens/total": 9496764, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 203351 + }, + { + "epoch": 9.84375, + "grad_norm": 0.12744079530239105, + "learning_rate": 8.467283740992458e-05, + "loss": 0.0011482161935418844, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00115, + "step": 630, + "tokens/total": 9511892, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 203673 + }, + { + "epoch": 9.859375, + "grad_norm": 0.0058466242626309395, + "learning_rate": 8.461818284739645e-05, + "loss": 0.00018700159853324294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00019, + "step": 631, + "tokens/total": 9526912, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 203992 + }, + { + "epoch": 9.875, + "grad_norm": 0.0028929621912539005, + "learning_rate": 8.456345109285131e-05, + "loss": 9.651956497691572e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 632, + "tokens/total": 9542048, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 204316 + }, + { + "epoch": 9.890625, + "grad_norm": 0.004049594979733229, + "learning_rate": 8.45086422889331e-05, + "loss": 0.00013334020331967622, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 633, + "tokens/total": 9557160, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 204640 + }, + { + "epoch": 9.90625, + "grad_norm": 0.004615152720361948, + "learning_rate": 8.445375657848657e-05, + "loss": 0.00019088154658675194, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00019, + "step": 634, + "tokens/total": 9572248, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 204962 + }, + { + "epoch": 9.921875, + "grad_norm": 0.014053276740014553, + "learning_rate": 8.439879410455693e-05, + "loss": 0.0004371690738480538, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00044, + "step": 635, + "tokens/total": 9587332, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 205286 + }, + { + "epoch": 9.9375, + "grad_norm": 0.003797621699050069, + "learning_rate": 8.434375501038944e-05, + "loss": 0.0001331718231085688, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00013, + "step": 636, + "tokens/total": 9602432, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 205611 + }, + { + "epoch": 9.953125, + "grad_norm": 0.017966385930776596, + "learning_rate": 8.428863943942904e-05, + "loss": 0.0004457358445506543, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00045, + "step": 637, + "tokens/total": 9617432, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 205935 + }, + { + "epoch": 9.96875, + "grad_norm": 0.0022921767085790634, + "learning_rate": 8.423344753531999e-05, + "loss": 9.555556607665494e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 638, + "tokens/total": 9632656, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 206254 + }, + { + "epoch": 9.984375, + "grad_norm": 0.005315504502505064, + "learning_rate": 8.417817944190553e-05, + "loss": 0.00016517250332981348, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00017, + "step": 639, + "tokens/total": 9647680, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 206574 + }, + { + "epoch": 10.0, + "grad_norm": 0.0037788504268974066, + "learning_rate": 8.412283530322736e-05, + "loss": 0.0001254509697901085, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00013, + "step": 640, + "tokens/total": 9662672, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 206900 + }, + { + "epoch": 10.015625, + "grad_norm": 0.010420114733278751, + "learning_rate": 8.406741526352553e-05, + "loss": 0.0003203490923624486, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00032, + "step": 641, + "tokens/total": 9677748, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 207225 + }, + { + "epoch": 10.03125, + "grad_norm": 0.00566550437361002, + "learning_rate": 8.401191946723776e-05, + "loss": 0.00013484270311892033, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 642, + "tokens/total": 9692824, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 207549 + }, + { + "epoch": 10.046875, + "grad_norm": 0.003279933473095298, + "learning_rate": 8.395634805899931e-05, + "loss": 9.422817674931139e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00009, + "step": 643, + "tokens/total": 9707908, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 207868 + }, + { + "epoch": 10.0625, + "grad_norm": 0.000977570889517665, + "learning_rate": 8.390070118364246e-05, + "loss": 4.221826384309679e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 644, + "tokens/total": 9722956, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 208190 + }, + { + "epoch": 10.078125, + "grad_norm": 0.0008564384770579636, + "learning_rate": 8.384497898619617e-05, + "loss": 3.925899727619253e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 645, + "tokens/total": 9738088, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 208514 + }, + { + "epoch": 10.09375, + "grad_norm": 0.0010848651872947812, + "learning_rate": 8.378918161188571e-05, + "loss": 5.050956315244548e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 646, + "tokens/total": 9753164, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 208839 + }, + { + "epoch": 10.109375, + "grad_norm": 0.000805368646979332, + "learning_rate": 8.373330920613229e-05, + "loss": 3.659196590888314e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 647, + "tokens/total": 9768176, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 209161 + }, + { + "epoch": 10.125, + "grad_norm": 0.0006011512014083564, + "learning_rate": 8.367736191455268e-05, + "loss": 2.033766395470593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 648, + "tokens/total": 9783276, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 209483 + }, + { + "epoch": 10.140625, + "grad_norm": 0.0013016491429880261, + "learning_rate": 8.362133988295879e-05, + "loss": 5.3827883675694466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 649, + "tokens/total": 9798340, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 209810 + }, + { + "epoch": 10.15625, + "grad_norm": 0.0025394551921635866, + "learning_rate": 8.356524325735735e-05, + "loss": 5.170019721845165e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 650, + "tokens/total": 9813404, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 210132 + }, + { + "epoch": 10.171875, + "grad_norm": 0.0010526948608458042, + "learning_rate": 8.35090721839495e-05, + "loss": 3.500043385429308e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 651, + "tokens/total": 9828520, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 210459 + }, + { + "epoch": 10.1875, + "grad_norm": 0.0006613961304537952, + "learning_rate": 8.345282680913033e-05, + "loss": 2.1081128579680808e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 652, + "tokens/total": 9843532, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 210785 + }, + { + "epoch": 10.203125, + "grad_norm": 0.0005401615053415298, + "learning_rate": 8.33965072794887e-05, + "loss": 2.4007260435610078e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 653, + "tokens/total": 9858592, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 211109 + }, + { + "epoch": 10.21875, + "grad_norm": 0.0007338224677368999, + "learning_rate": 8.334011374180668e-05, + "loss": 2.9307986551430076e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 654, + "tokens/total": 9873660, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 211432 + }, + { + "epoch": 10.234375, + "grad_norm": 0.0006001188303343952, + "learning_rate": 8.328364634305917e-05, + "loss": 1.935281397891231e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 655, + "tokens/total": 9888736, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 211754 + }, + { + "epoch": 10.25, + "grad_norm": 0.0027960201259702444, + "learning_rate": 8.322710523041366e-05, + "loss": 5.040257019572891e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 656, + "tokens/total": 9903756, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 212075 + }, + { + "epoch": 10.265625, + "grad_norm": 0.0007850946276448667, + "learning_rate": 8.317049055122971e-05, + "loss": 2.9081335014780052e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 657, + "tokens/total": 9918800, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 212399 + }, + { + "epoch": 10.28125, + "grad_norm": 1.357591986656189, + "learning_rate": 8.311380245305863e-05, + "loss": 0.008083355613052845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00812, + "step": 658, + "tokens/total": 9933768, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 212718 + }, + { + "epoch": 10.296875, + "grad_norm": 0.005757445003837347, + "learning_rate": 8.305704108364301e-05, + "loss": 9.082010365091264e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00009, + "step": 659, + "tokens/total": 9948864, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 213040 + }, + { + "epoch": 10.3125, + "grad_norm": 0.0005670920945703983, + "learning_rate": 8.30002065909165e-05, + "loss": 2.1615041987388395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 660, + "tokens/total": 9964024, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 213361 + }, + { + "epoch": 10.328125, + "grad_norm": 0.00048100232379511, + "learning_rate": 8.294329912300329e-05, + "loss": 1.8358683519181795e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 661, + "tokens/total": 9979200, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 213682 + }, + { + "epoch": 10.34375, + "grad_norm": 0.0493755005300045, + "learning_rate": 8.288631882821773e-05, + "loss": 0.00028375795227475464, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00028, + "step": 662, + "tokens/total": 9994184, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 214002 + }, + { + "epoch": 10.359375, + "grad_norm": 0.7679035663604736, + "learning_rate": 8.282926585506399e-05, + "loss": 0.005704966373741627, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00572, + "step": 663, + "tokens/total": 10009244, + "tokens/train_per_sec_per_gpu": 8.94, + "tokens/trainable": 214318 + }, + { + "epoch": 10.375, + "grad_norm": 0.0005227076471783221, + "learning_rate": 8.277214035223568e-05, + "loss": 1.6632866390864365e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 664, + "tokens/total": 10024264, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 214647 + }, + { + "epoch": 10.390625, + "grad_norm": 0.0003327404265291989, + "learning_rate": 8.271494246861545e-05, + "loss": 1.3077985386189539e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 665, + "tokens/total": 10039468, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 214968 + }, + { + "epoch": 10.40625, + "grad_norm": 0.0005173759418539703, + "learning_rate": 8.26576723532745e-05, + "loss": 1.63102140504634e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 666, + "tokens/total": 10054640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 215291 + }, + { + "epoch": 10.421875, + "grad_norm": 0.0021957808639854193, + "learning_rate": 8.260033015547241e-05, + "loss": 3.234932592022233e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 667, + "tokens/total": 10069820, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 215612 + }, + { + "epoch": 10.4375, + "grad_norm": 0.0004071689327247441, + "learning_rate": 8.254291602465653e-05, + "loss": 1.577117836859543e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 668, + "tokens/total": 10084928, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 215940 + }, + { + "epoch": 10.453125, + "grad_norm": 0.0003882338060066104, + "learning_rate": 8.248543011046174e-05, + "loss": 1.6156058336491697e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 669, + "tokens/total": 10099972, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 216264 + }, + { + "epoch": 10.46875, + "grad_norm": 0.00012573694402817637, + "learning_rate": 8.242787256270995e-05, + "loss": 4.233596882841084e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 670, + "tokens/total": 10115120, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 216588 + }, + { + "epoch": 10.484375, + "grad_norm": 0.10736624896526337, + "learning_rate": 8.237024353140983e-05, + "loss": 0.00040273292688652873, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0004, + "step": 671, + "tokens/total": 10130164, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 216911 + }, + { + "epoch": 10.5, + "grad_norm": 0.0005355168250389397, + "learning_rate": 8.231254316675629e-05, + "loss": 1.6170764865819365e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 672, + "tokens/total": 10145220, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 217239 + }, + { + "epoch": 10.515625, + "grad_norm": 0.013825232163071632, + "learning_rate": 8.22547716191302e-05, + "loss": 0.00011374444875400513, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00011, + "step": 673, + "tokens/total": 10160416, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 217566 + }, + { + "epoch": 10.53125, + "grad_norm": 0.6939883828163147, + "learning_rate": 8.219692903909794e-05, + "loss": 0.0002577207633294165, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00026, + "step": 674, + "tokens/total": 10175636, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 217890 + }, + { + "epoch": 10.546875, + "grad_norm": 0.006983619183301926, + "learning_rate": 8.213901557741098e-05, + "loss": 5.019552554585971e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 675, + "tokens/total": 10190752, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 218218 + }, + { + "epoch": 10.5625, + "grad_norm": 0.0011591928778216243, + "learning_rate": 8.208103138500557e-05, + "loss": 2.6914578484138474e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 676, + "tokens/total": 10205880, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 218539 + }, + { + "epoch": 10.578125, + "grad_norm": 0.00047723689931444824, + "learning_rate": 8.202297661300228e-05, + "loss": 1.389540466334438e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 677, + "tokens/total": 10220984, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 218861 + }, + { + "epoch": 10.59375, + "grad_norm": 0.018745990470051765, + "learning_rate": 8.196485141270563e-05, + "loss": 0.00020080397371202707, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0002, + "step": 678, + "tokens/total": 10236212, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 219186 + }, + { + "epoch": 10.609375, + "grad_norm": 0.0008061480475589633, + "learning_rate": 8.190665593560367e-05, + "loss": 2.657961522345431e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 679, + "tokens/total": 10251296, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 219510 + }, + { + "epoch": 10.625, + "grad_norm": 0.00017094668874051422, + "learning_rate": 8.184839033336767e-05, + "loss": 5.520010745385662e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 680, + "tokens/total": 10266344, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 219834 + }, + { + "epoch": 10.640625, + "grad_norm": 0.00044193342910148203, + "learning_rate": 8.179005475785162e-05, + "loss": 1.244948543899227e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 681, + "tokens/total": 10281380, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 220156 + }, + { + "epoch": 10.65625, + "grad_norm": 0.0012106681242585182, + "learning_rate": 8.173164936109184e-05, + "loss": 3.160563937854022e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 682, + "tokens/total": 10296504, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 220479 + }, + { + "epoch": 10.671875, + "grad_norm": 0.0003608819970395416, + "learning_rate": 8.167317429530669e-05, + "loss": 1.3102995580993593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 683, + "tokens/total": 10311544, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 220803 + }, + { + "epoch": 10.6875, + "grad_norm": 0.0006010961951687932, + "learning_rate": 8.161462971289608e-05, + "loss": 1.8992457626154646e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 684, + "tokens/total": 10326644, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 221124 + }, + { + "epoch": 10.703125, + "grad_norm": 0.000617502664681524, + "learning_rate": 8.155601576644108e-05, + "loss": 1.7654312614467926e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 685, + "tokens/total": 10341824, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 221449 + }, + { + "epoch": 10.71875, + "grad_norm": 0.00039724848465994, + "learning_rate": 8.14973326087036e-05, + "loss": 1.1104682926088572e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 686, + "tokens/total": 10357076, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 221774 + }, + { + "epoch": 10.734375, + "grad_norm": 0.0011247704969719052, + "learning_rate": 8.14385803926258e-05, + "loss": 2.0452473108889535e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 687, + "tokens/total": 10372192, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 222094 + }, + { + "epoch": 10.75, + "grad_norm": 0.00044882961083203554, + "learning_rate": 8.137975927132995e-05, + "loss": 1.5210380297503434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 688, + "tokens/total": 10387336, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 222419 + }, + { + "epoch": 10.765625, + "grad_norm": 0.000701107201166451, + "learning_rate": 8.132086939811787e-05, + "loss": 2.3597141989739612e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 689, + "tokens/total": 10402484, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 222742 + }, + { + "epoch": 10.78125, + "grad_norm": 0.0005961813731119037, + "learning_rate": 8.126191092647053e-05, + "loss": 1.2273942957108375e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 690, + "tokens/total": 10417640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 223062 + }, + { + "epoch": 10.796875, + "grad_norm": 0.00022672704653814435, + "learning_rate": 8.120288401004771e-05, + "loss": 7.491194082831498e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 691, + "tokens/total": 10432700, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 223383 + }, + { + "epoch": 10.8125, + "grad_norm": 0.0015177978202700615, + "learning_rate": 8.114378880268758e-05, + "loss": 2.7457914256956428e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 692, + "tokens/total": 10447872, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 223707 + }, + { + "epoch": 10.828125, + "grad_norm": 0.0003037489950656891, + "learning_rate": 8.108462545840627e-05, + "loss": 9.898707503452897e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 693, + "tokens/total": 10462960, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 224031 + }, + { + "epoch": 10.84375, + "grad_norm": 0.0010784813202917576, + "learning_rate": 8.102539413139751e-05, + "loss": 2.49961631197948e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 694, + "tokens/total": 10477964, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 224352 + }, + { + "epoch": 10.859375, + "grad_norm": 0.0007603645790368319, + "learning_rate": 8.09660949760322e-05, + "loss": 1.2760282515955623e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 695, + "tokens/total": 10493088, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 224677 + }, + { + "epoch": 10.875, + "grad_norm": 0.0005472145276144147, + "learning_rate": 8.090672814685802e-05, + "loss": 1.6190988390007988e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 696, + "tokens/total": 10508160, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 225001 + }, + { + "epoch": 10.890625, + "grad_norm": 0.00033579772571101785, + "learning_rate": 8.084729379859902e-05, + "loss": 1.1744982657546643e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 697, + "tokens/total": 10523108, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 225323 + }, + { + "epoch": 10.90625, + "grad_norm": 0.0003757970698643476, + "learning_rate": 8.078779208615524e-05, + "loss": 1.397086543875048e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 698, + "tokens/total": 10538160, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 225645 + }, + { + "epoch": 10.921875, + "grad_norm": 0.0003279249358456582, + "learning_rate": 8.072822316460223e-05, + "loss": 1.0247182217426598e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 699, + "tokens/total": 10553296, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 225974 + }, + { + "epoch": 10.9375, + "grad_norm": 0.00023233289539348334, + "learning_rate": 8.066858718919078e-05, + "loss": 7.159566393966088e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 700, + "tokens/total": 10568452, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 226296 + }, + { + "epoch": 10.953125, + "grad_norm": 0.00015506023191846907, + "learning_rate": 8.060888431534638e-05, + "loss": 5.614952442556387e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 701, + "tokens/total": 10583552, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 226619 + }, + { + "epoch": 10.96875, + "grad_norm": 0.0003310837200842798, + "learning_rate": 8.054911469866894e-05, + "loss": 1.0138256584468763e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 702, + "tokens/total": 10598668, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 226944 + }, + { + "epoch": 10.984375, + "grad_norm": 0.0005130712524987757, + "learning_rate": 8.048927849493222e-05, + "loss": 3.6420531159819802e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 703, + "tokens/total": 10613828, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 227269 + }, + { + "epoch": 11.0, + "grad_norm": 0.0004405453510116786, + "learning_rate": 8.042937586008359e-05, + "loss": 8.709830581210554e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 704, + "tokens/total": 10628960, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 227590 + }, + { + "epoch": 11.015625, + "grad_norm": 0.00013603588740807027, + "learning_rate": 8.036940695024354e-05, + "loss": 4.3405211727076676e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 705, + "tokens/total": 10644044, + "tokens/train_per_sec_per_gpu": 8.7, + "tokens/trainable": 227915 + }, + { + "epoch": 11.03125, + "grad_norm": 0.0004189728060737252, + "learning_rate": 8.030937192170529e-05, + "loss": 1.2250530744495336e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 706, + "tokens/total": 10659112, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 228237 + }, + { + "epoch": 11.046875, + "grad_norm": 0.00040614636964164674, + "learning_rate": 8.02492709309344e-05, + "loss": 6.656000550719909e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 707, + "tokens/total": 10674172, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 228567 + }, + { + "epoch": 11.0625, + "grad_norm": 0.0004786802746821195, + "learning_rate": 8.018910413456828e-05, + "loss": 6.218393536983058e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 708, + "tokens/total": 10689384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 228891 + }, + { + "epoch": 11.078125, + "grad_norm": 0.00014954940706957132, + "learning_rate": 8.012887168941592e-05, + "loss": 5.242945462669013e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 709, + "tokens/total": 10704556, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 229221 + }, + { + "epoch": 11.09375, + "grad_norm": 0.0003377373795956373, + "learning_rate": 8.006857375245736e-05, + "loss": 9.90279204415856e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 710, + "tokens/total": 10719640, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 229540 + }, + { + "epoch": 11.109375, + "grad_norm": 0.0003592268913052976, + "learning_rate": 8.000821048084334e-05, + "loss": 7.273632945725694e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 711, + "tokens/total": 10734776, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 229861 + }, + { + "epoch": 11.125, + "grad_norm": 0.00023520697141066194, + "learning_rate": 7.994778203189489e-05, + "loss": 6.704783572786255e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 712, + "tokens/total": 10749824, + "tokens/train_per_sec_per_gpu": 8.74, + "tokens/trainable": 230186 + }, + { + "epoch": 11.140625, + "grad_norm": 0.00010122328239958733, + "learning_rate": 7.988728856310289e-05, + "loss": 2.9637367333634757e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 713, + "tokens/total": 10764920, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 230511 + }, + { + "epoch": 11.15625, + "grad_norm": 0.00035993458004668355, + "learning_rate": 7.982673023212767e-05, + "loss": 1.1683331649692263e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 714, + "tokens/total": 10780068, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 230834 + }, + { + "epoch": 11.171875, + "grad_norm": 0.00016197813965845853, + "learning_rate": 7.976610719679865e-05, + "loss": 4.324124347476754e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 715, + "tokens/total": 10795172, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 231159 + }, + { + "epoch": 11.1875, + "grad_norm": 0.00017355824820697308, + "learning_rate": 7.970541961511382e-05, + "loss": 4.932363935949979e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 716, + "tokens/total": 10810260, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 231480 + }, + { + "epoch": 11.203125, + "grad_norm": 0.0003116214938927442, + "learning_rate": 7.964466764523945e-05, + "loss": 8.68427196110133e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 717, + "tokens/total": 10825452, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 231806 + }, + { + "epoch": 11.21875, + "grad_norm": 3.7266719341278076, + "learning_rate": 7.958385144550958e-05, + "loss": 0.008427927270531654, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00846, + "step": 718, + "tokens/total": 10840596, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 232128 + }, + { + "epoch": 11.234375, + "grad_norm": 0.0003720409295056015, + "learning_rate": 7.95229711744257e-05, + "loss": 6.924952231202042e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 719, + "tokens/total": 10855700, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 232446 + }, + { + "epoch": 11.25, + "grad_norm": 4.757077693939209, + "learning_rate": 7.946202699065622e-05, + "loss": 0.005191232543438673, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0052, + "step": 720, + "tokens/total": 10870892, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 232771 + }, + { + "epoch": 11.265625, + "grad_norm": 0.00044851572602055967, + "learning_rate": 7.940101905303615e-05, + "loss": 1.3887852219340857e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 721, + "tokens/total": 10885960, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 233096 + }, + { + "epoch": 11.28125, + "grad_norm": 0.00026496979990042746, + "learning_rate": 7.93399475205667e-05, + "loss": 7.952117812237702e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 722, + "tokens/total": 10900976, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 233421 + }, + { + "epoch": 11.296875, + "grad_norm": 0.0004989475710317492, + "learning_rate": 7.927881255241474e-05, + "loss": 1.2574350876093376e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 723, + "tokens/total": 10916092, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 233746 + }, + { + "epoch": 11.3125, + "grad_norm": 0.00041149379103444517, + "learning_rate": 7.921761430791252e-05, + "loss": 1.3969710380479228e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 724, + "tokens/total": 10931228, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 234068 + }, + { + "epoch": 11.328125, + "grad_norm": 1.1542341709136963, + "learning_rate": 7.915635294655723e-05, + "loss": 0.014514163136482239, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01462, + "step": 725, + "tokens/total": 10946452, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 234389 + }, + { + "epoch": 11.34375, + "grad_norm": 0.0007863897481001914, + "learning_rate": 7.909502862801049e-05, + "loss": 2.557808329584077e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 726, + "tokens/total": 10961580, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 234715 + }, + { + "epoch": 11.359375, + "grad_norm": 0.001132657052949071, + "learning_rate": 7.903364151209804e-05, + "loss": 3.360592745593749e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 727, + "tokens/total": 10976596, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 235034 + }, + { + "epoch": 11.375, + "grad_norm": 0.0007901990320533514, + "learning_rate": 7.897219175880931e-05, + "loss": 3.233204915886745e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 728, + "tokens/total": 10991728, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 235355 + }, + { + "epoch": 11.390625, + "grad_norm": 0.0005902523989789188, + "learning_rate": 7.89106795282969e-05, + "loss": 2.2109001292847097e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 729, + "tokens/total": 11006712, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 235674 + }, + { + "epoch": 11.40625, + "grad_norm": 0.012264137156307697, + "learning_rate": 7.884910498087632e-05, + "loss": 0.00024805692373774946, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00025, + "step": 730, + "tokens/total": 11021724, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 235999 + }, + { + "epoch": 11.421875, + "grad_norm": 0.026404112577438354, + "learning_rate": 7.878746827702545e-05, + "loss": 0.00025394666590727866, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00025, + "step": 731, + "tokens/total": 11036780, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 236322 + }, + { + "epoch": 11.4375, + "grad_norm": 0.05578846484422684, + "learning_rate": 7.872576957738418e-05, + "loss": 0.0004502359952311963, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00045, + "step": 732, + "tokens/total": 11051868, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 236642 + }, + { + "epoch": 11.453125, + "grad_norm": 0.007219335995614529, + "learning_rate": 7.866400904275396e-05, + "loss": 0.00013881822815164924, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 733, + "tokens/total": 11066984, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 236964 + }, + { + "epoch": 11.46875, + "grad_norm": 0.04276825115084648, + "learning_rate": 7.860218683409742e-05, + "loss": 0.00031288809259422123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 734, + "tokens/total": 11082000, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 237291 + }, + { + "epoch": 11.484375, + "grad_norm": 0.13936324417591095, + "learning_rate": 7.854030311253792e-05, + "loss": 0.0022737535182386637, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00228, + "step": 735, + "tokens/total": 11097164, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 237614 + }, + { + "epoch": 11.5, + "grad_norm": 0.004859534092247486, + "learning_rate": 7.847835803935911e-05, + "loss": 5.4845815611770377e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 736, + "tokens/total": 11112288, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 237939 + }, + { + "epoch": 11.515625, + "grad_norm": 0.00597998034209013, + "learning_rate": 7.841635177600457e-05, + "loss": 0.0001230256602866575, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00012, + "step": 737, + "tokens/total": 11127476, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 238268 + }, + { + "epoch": 11.53125, + "grad_norm": 0.07212433218955994, + "learning_rate": 7.835428448407737e-05, + "loss": 0.0007460076012648642, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00075, + "step": 738, + "tokens/total": 11142488, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 238592 + }, + { + "epoch": 11.546875, + "grad_norm": 0.0024873202200978994, + "learning_rate": 7.829215632533957e-05, + "loss": 7.913773879408836e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00008, + "step": 739, + "tokens/total": 11157532, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 238916 + }, + { + "epoch": 11.5625, + "grad_norm": 1.5609562397003174, + "learning_rate": 7.822996746171192e-05, + "loss": 0.014905812218785286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01502, + "step": 740, + "tokens/total": 11172620, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 239235 + }, + { + "epoch": 11.578125, + "grad_norm": 0.0014489549212157726, + "learning_rate": 7.816771805527336e-05, + "loss": 6.693808973068371e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00007, + "step": 741, + "tokens/total": 11187780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 239557 + }, + { + "epoch": 11.59375, + "grad_norm": 0.008130564354360104, + "learning_rate": 7.810540826826064e-05, + "loss": 9.863007289823145e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 742, + "tokens/total": 11202908, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 239884 + }, + { + "epoch": 11.609375, + "grad_norm": 0.5576651096343994, + "learning_rate": 7.804303826306784e-05, + "loss": 0.0035350474063307047, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00354, + "step": 743, + "tokens/total": 11218028, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 240200 + }, + { + "epoch": 11.625, + "grad_norm": 0.0007206861046142876, + "learning_rate": 7.798060820224602e-05, + "loss": 3.09372044284828e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 744, + "tokens/total": 11233028, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 240521 + }, + { + "epoch": 11.640625, + "grad_norm": 0.002179693430662155, + "learning_rate": 7.791811824850271e-05, + "loss": 7.596659997943789e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 745, + "tokens/total": 11248200, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 240844 + }, + { + "epoch": 11.65625, + "grad_norm": 0.0010591476457193494, + "learning_rate": 7.78555685647016e-05, + "loss": 3.789784386754036e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 746, + "tokens/total": 11263204, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 241171 + }, + { + "epoch": 11.671875, + "grad_norm": 0.0022162902168929577, + "learning_rate": 7.779295931386202e-05, + "loss": 5.531610077014193e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 747, + "tokens/total": 11278288, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 241493 + }, + { + "epoch": 11.6875, + "grad_norm": 0.0014585545286536217, + "learning_rate": 7.773029065915851e-05, + "loss": 4.745048499898985e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 748, + "tokens/total": 11293364, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 241821 + }, + { + "epoch": 11.703125, + "grad_norm": 0.0019064174266532063, + "learning_rate": 7.766756276392049e-05, + "loss": 5.8749526942847297e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00006, + "step": 749, + "tokens/total": 11308524, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 242145 + }, + { + "epoch": 11.71875, + "grad_norm": 0.0029175870586186647, + "learning_rate": 7.760477579163175e-05, + "loss": 6.556163134519011e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00007, + "step": 750, + "tokens/total": 11323636, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 242475 + }, + { + "epoch": 11.734375, + "grad_norm": 0.1474231481552124, + "learning_rate": 7.754192990593006e-05, + "loss": 0.0007740260916762054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00077, + "step": 751, + "tokens/total": 11338712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 242796 + }, + { + "epoch": 11.75, + "grad_norm": 0.0071921986527740955, + "learning_rate": 7.747902527060669e-05, + "loss": 0.00010170727910008281, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 752, + "tokens/total": 11353776, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 243120 + }, + { + "epoch": 11.765625, + "grad_norm": 0.0011470766039565206, + "learning_rate": 7.741606204960606e-05, + "loss": 3.4301298001082614e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 753, + "tokens/total": 11368904, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 243445 + }, + { + "epoch": 11.78125, + "grad_norm": 0.0019251713529229164, + "learning_rate": 7.735304040702528e-05, + "loss": 5.888251689611934e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 754, + "tokens/total": 11384068, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 243769 + }, + { + "epoch": 11.796875, + "grad_norm": 0.0029021066147834063, + "learning_rate": 7.728996050711372e-05, + "loss": 5.249957393971272e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 755, + "tokens/total": 11399116, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 244090 + }, + { + "epoch": 11.8125, + "grad_norm": 0.0010381685569882393, + "learning_rate": 7.722682251427256e-05, + "loss": 2.9339025786612183e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 756, + "tokens/total": 11414248, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 244411 + }, + { + "epoch": 11.828125, + "grad_norm": 0.0008667893125675619, + "learning_rate": 7.716362659305444e-05, + "loss": 3.0496574254357256e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 757, + "tokens/total": 11429368, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 244739 + }, + { + "epoch": 11.84375, + "grad_norm": 0.00029208004707470536, + "learning_rate": 7.710037290816289e-05, + "loss": 1.4064591596252285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 758, + "tokens/total": 11444340, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 245060 + }, + { + "epoch": 11.859375, + "grad_norm": 0.0008153665694408119, + "learning_rate": 7.703706162445204e-05, + "loss": 2.3803997464710847e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 759, + "tokens/total": 11459428, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 245383 + }, + { + "epoch": 11.875, + "grad_norm": 0.001724294270388782, + "learning_rate": 7.697369290692617e-05, + "loss": 2.3616061298525892e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 760, + "tokens/total": 11474592, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 245704 + }, + { + "epoch": 11.890625, + "grad_norm": 0.0006179643096402287, + "learning_rate": 7.691026692073915e-05, + "loss": 2.2060095943743363e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 761, + "tokens/total": 11489644, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 246026 + }, + { + "epoch": 11.90625, + "grad_norm": 0.4296891689300537, + "learning_rate": 7.68467838311942e-05, + "loss": 0.021270249038934708, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0215, + "step": 762, + "tokens/total": 11504716, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 246349 + }, + { + "epoch": 11.921875, + "grad_norm": 0.000874324468895793, + "learning_rate": 7.678324380374328e-05, + "loss": 3.0648101528640836e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 763, + "tokens/total": 11519828, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 246668 + }, + { + "epoch": 11.9375, + "grad_norm": 0.0005559792625717819, + "learning_rate": 7.671964700398684e-05, + "loss": 2.257221785839647e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 764, + "tokens/total": 11534864, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 246991 + }, + { + "epoch": 11.953125, + "grad_norm": 0.0011306314263492823, + "learning_rate": 7.66559935976732e-05, + "loss": 4.120477387914434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 765, + "tokens/total": 11549908, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 247312 + }, + { + "epoch": 11.96875, + "grad_norm": 0.00738960737362504, + "learning_rate": 7.659228375069827e-05, + "loss": 0.00011856225319206715, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 766, + "tokens/total": 11565044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 247636 + }, + { + "epoch": 11.984375, + "grad_norm": 0.012207427993416786, + "learning_rate": 7.652851762910505e-05, + "loss": 0.0002129970962414518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00021, + "step": 767, + "tokens/total": 11580188, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 247959 + }, + { + "epoch": 12.0, + "grad_norm": 0.005771435797214508, + "learning_rate": 7.646469539908316e-05, + "loss": 0.0001359613670501858, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 768, + "tokens/total": 11595328, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 248280 + }, + { + "epoch": 12.015625, + "grad_norm": 0.16021107137203217, + "learning_rate": 7.640081722696851e-05, + "loss": 0.002986999461427331, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00299, + "step": 769, + "tokens/total": 11610408, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 248603 + }, + { + "epoch": 12.03125, + "grad_norm": 0.06659352034330368, + "learning_rate": 7.633688327924279e-05, + "loss": 0.00023177383991423994, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00023, + "step": 770, + "tokens/total": 11625440, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 248929 + }, + { + "epoch": 12.046875, + "grad_norm": 0.043171677738428116, + "learning_rate": 7.627289372253304e-05, + "loss": 0.0004853203718084842, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00049, + "step": 771, + "tokens/total": 11640488, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 249250 + }, + { + "epoch": 12.0625, + "grad_norm": 0.06471310555934906, + "learning_rate": 7.620884872361122e-05, + "loss": 0.00033691420685499907, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00034, + "step": 772, + "tokens/total": 11655628, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 249573 + }, + { + "epoch": 12.078125, + "grad_norm": 0.0012351584155112505, + "learning_rate": 7.61447484493938e-05, + "loss": 4.9392016080673784e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 773, + "tokens/total": 11670680, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 249895 + }, + { + "epoch": 12.09375, + "grad_norm": 0.012346632778644562, + "learning_rate": 7.608059306694137e-05, + "loss": 0.00012409746705088764, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00012, + "step": 774, + "tokens/total": 11685696, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 250222 + }, + { + "epoch": 12.109375, + "grad_norm": 0.0019148180726915598, + "learning_rate": 7.601638274345803e-05, + "loss": 7.570564048364758e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 775, + "tokens/total": 11700828, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 250546 + }, + { + "epoch": 12.125, + "grad_norm": 0.011315557174384594, + "learning_rate": 7.595211764629117e-05, + "loss": 7.948732672957703e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 776, + "tokens/total": 11715968, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 250869 + }, + { + "epoch": 12.140625, + "grad_norm": 0.0014873143518343568, + "learning_rate": 7.588779794293086e-05, + "loss": 5.8105400967178866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 777, + "tokens/total": 11731076, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 251195 + }, + { + "epoch": 12.15625, + "grad_norm": 0.0056472280994057655, + "learning_rate": 7.582342380100953e-05, + "loss": 5.892526314710267e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 778, + "tokens/total": 11746184, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 251521 + }, + { + "epoch": 12.171875, + "grad_norm": 0.007838607765734196, + "learning_rate": 7.575899538830149e-05, + "loss": 0.00010413450945634395, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 779, + "tokens/total": 11761240, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 251840 + }, + { + "epoch": 12.1875, + "grad_norm": 0.046443358063697815, + "learning_rate": 7.569451287272247e-05, + "loss": 0.0003978235472459346, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0004, + "step": 780, + "tokens/total": 11776172, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 252166 + }, + { + "epoch": 12.203125, + "grad_norm": 0.001991559285670519, + "learning_rate": 7.562997642232921e-05, + "loss": 7.929989078547806e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 781, + "tokens/total": 11791252, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 252493 + }, + { + "epoch": 12.21875, + "grad_norm": 0.0033611422404646873, + "learning_rate": 7.556538620531904e-05, + "loss": 7.862138591008261e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 782, + "tokens/total": 11806280, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 252815 + }, + { + "epoch": 12.234375, + "grad_norm": 0.002292010700330138, + "learning_rate": 7.55007423900294e-05, + "loss": 7.994483166839927e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 783, + "tokens/total": 11821296, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 253138 + }, + { + "epoch": 12.25, + "grad_norm": 0.0009671649895608425, + "learning_rate": 7.54360451449374e-05, + "loss": 4.199397881166078e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 784, + "tokens/total": 11836420, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 253459 + }, + { + "epoch": 12.265625, + "grad_norm": 0.0023980247788131237, + "learning_rate": 7.537129463865945e-05, + "loss": 7.811621617292985e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00008, + "step": 785, + "tokens/total": 11851556, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 253785 + }, + { + "epoch": 12.28125, + "grad_norm": 0.005567625630646944, + "learning_rate": 7.530649103995074e-05, + "loss": 0.00014928210293874145, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00015, + "step": 786, + "tokens/total": 11866676, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 254111 + }, + { + "epoch": 12.296875, + "grad_norm": 0.005312805995345116, + "learning_rate": 7.524163451770481e-05, + "loss": 0.00014692469267174602, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00015, + "step": 787, + "tokens/total": 11881856, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 254433 + }, + { + "epoch": 12.3125, + "grad_norm": 0.005031264387071133, + "learning_rate": 7.517672524095319e-05, + "loss": 8.916858496377245e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 788, + "tokens/total": 11897048, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 254756 + }, + { + "epoch": 12.328125, + "grad_norm": 0.0007357180002145469, + "learning_rate": 7.511176337886485e-05, + "loss": 3.350479164510034e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00003, + "step": 789, + "tokens/total": 11912072, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 255074 + }, + { + "epoch": 12.34375, + "grad_norm": 0.0009413110092282295, + "learning_rate": 7.50467491007458e-05, + "loss": 3.6423676647245884e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 790, + "tokens/total": 11927120, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 255399 + }, + { + "epoch": 12.359375, + "grad_norm": 0.0008683132473379374, + "learning_rate": 7.498168257603875e-05, + "loss": 3.2453805033583194e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 791, + "tokens/total": 11942172, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 255723 + }, + { + "epoch": 12.375, + "grad_norm": 0.001063066185452044, + "learning_rate": 7.491656397432247e-05, + "loss": 2.9047278076177463e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 792, + "tokens/total": 11957288, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 256046 + }, + { + "epoch": 12.390625, + "grad_norm": 0.0013455420266836882, + "learning_rate": 7.485139346531149e-05, + "loss": 4.68404614366591e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 793, + "tokens/total": 11972424, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 256368 + }, + { + "epoch": 12.40625, + "grad_norm": 0.0037693495396524668, + "learning_rate": 7.478617121885563e-05, + "loss": 6.180455966386944e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 794, + "tokens/total": 11987508, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 256692 + }, + { + "epoch": 12.421875, + "grad_norm": 0.0008823846001178026, + "learning_rate": 7.472089740493959e-05, + "loss": 3.486638888716698e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 795, + "tokens/total": 12002564, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 257012 + }, + { + "epoch": 12.4375, + "grad_norm": 0.0025711311027407646, + "learning_rate": 7.46555721936824e-05, + "loss": 6.127560482127592e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 796, + "tokens/total": 12017704, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 257339 + }, + { + "epoch": 12.453125, + "grad_norm": 0.020793205127120018, + "learning_rate": 7.459019575533707e-05, + "loss": 5.980060814181343e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 797, + "tokens/total": 12032856, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 257664 + }, + { + "epoch": 12.46875, + "grad_norm": 0.0010502856457605958, + "learning_rate": 7.452476826029011e-05, + "loss": 3.661629307316616e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00004, + "step": 798, + "tokens/total": 12048028, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 257988 + }, + { + "epoch": 12.484375, + "grad_norm": 0.0005576885305345058, + "learning_rate": 7.445928987906114e-05, + "loss": 2.2148260541143827e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00002, + "step": 799, + "tokens/total": 12063092, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 258315 + }, + { + "epoch": 12.5, + "grad_norm": 0.0005081727867946029, + "learning_rate": 7.439376078230234e-05, + "loss": 2.0269433662178926e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 800, + "tokens/total": 12078208, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 258638 + }, + { + "epoch": 12.515625, + "grad_norm": 0.0006259866640903056, + "learning_rate": 7.432818114079813e-05, + "loss": 2.4334258341696113e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 801, + "tokens/total": 12093348, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 258963 + }, + { + "epoch": 12.53125, + "grad_norm": 0.0003684913972392678, + "learning_rate": 7.426255112546458e-05, + "loss": 1.6602209143457003e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 802, + "tokens/total": 12108412, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 259287 + }, + { + "epoch": 12.546875, + "grad_norm": 0.0004341568856034428, + "learning_rate": 7.419687090734912e-05, + "loss": 1.815272480598651e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 803, + "tokens/total": 12123480, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 259607 + }, + { + "epoch": 12.5625, + "grad_norm": 0.008828886784613132, + "learning_rate": 7.413114065762999e-05, + "loss": 7.399358582915738e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00007, + "step": 804, + "tokens/total": 12138644, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 259933 + }, + { + "epoch": 12.578125, + "grad_norm": 0.0007751079392619431, + "learning_rate": 7.406536054761584e-05, + "loss": 1.663630609982647e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 805, + "tokens/total": 12153664, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 260256 + }, + { + "epoch": 12.59375, + "grad_norm": 0.00035115418722853065, + "learning_rate": 7.399953074874523e-05, + "loss": 1.597570371814072e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 806, + "tokens/total": 12168872, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 260582 + }, + { + "epoch": 12.609375, + "grad_norm": 0.0002732598513830453, + "learning_rate": 7.393365143258626e-05, + "loss": 1.3219183529145084e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 807, + "tokens/total": 12184012, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 260909 + }, + { + "epoch": 12.625, + "grad_norm": 0.0003034104302059859, + "learning_rate": 7.386772277083606e-05, + "loss": 1.4620679394283798e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 808, + "tokens/total": 12198968, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 261237 + }, + { + "epoch": 12.640625, + "grad_norm": 0.00048796235932968557, + "learning_rate": 7.38017449353204e-05, + "loss": 1.7275449863518588e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 809, + "tokens/total": 12214048, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 261560 + }, + { + "epoch": 12.65625, + "grad_norm": 0.0005510294577106833, + "learning_rate": 7.373571809799319e-05, + "loss": 1.772215728124138e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 810, + "tokens/total": 12229144, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 261882 + }, + { + "epoch": 12.671875, + "grad_norm": 0.0004420862242113799, + "learning_rate": 7.3669642430936e-05, + "loss": 1.773734766175039e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 811, + "tokens/total": 12244368, + "tokens/train_per_sec_per_gpu": 8.85, + "tokens/trainable": 262199 + }, + { + "epoch": 12.6875, + "grad_norm": 0.00022525829263031483, + "learning_rate": 7.360351810635774e-05, + "loss": 1.1139311936858576e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 812, + "tokens/total": 12259492, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 262525 + }, + { + "epoch": 12.703125, + "grad_norm": 0.00035630102502182126, + "learning_rate": 7.353734529659409e-05, + "loss": 1.4786606698180549e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 813, + "tokens/total": 12274608, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 262850 + }, + { + "epoch": 12.71875, + "grad_norm": 0.0002704367507249117, + "learning_rate": 7.347112417410712e-05, + "loss": 1.203708779939916e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 814, + "tokens/total": 12289780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 263171 + }, + { + "epoch": 12.734375, + "grad_norm": 0.0003859064891003072, + "learning_rate": 7.340485491148478e-05, + "loss": 1.5924519175314344e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 815, + "tokens/total": 12304880, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 263495 + }, + { + "epoch": 12.75, + "grad_norm": 0.00031393321114592254, + "learning_rate": 7.33385376814405e-05, + "loss": 1.391779187542852e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 816, + "tokens/total": 12319984, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 263818 + }, + { + "epoch": 12.765625, + "grad_norm": 0.0002688078675419092, + "learning_rate": 7.327217265681276e-05, + "loss": 1.0123962056240998e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 817, + "tokens/total": 12335172, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 264139 + }, + { + "epoch": 12.78125, + "grad_norm": 0.00039067966281436384, + "learning_rate": 7.320576001056452e-05, + "loss": 1.2487973435781896e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 818, + "tokens/total": 12350312, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 264466 + }, + { + "epoch": 12.796875, + "grad_norm": 0.0007766951457597315, + "learning_rate": 7.313929991578294e-05, + "loss": 2.187816789955832e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 819, + "tokens/total": 12365496, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 264789 + }, + { + "epoch": 12.8125, + "grad_norm": 0.004195546731352806, + "learning_rate": 7.307279254567882e-05, + "loss": 2.1917332560406066e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 820, + "tokens/total": 12380560, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 265112 + }, + { + "epoch": 12.828125, + "grad_norm": 0.000822481291834265, + "learning_rate": 7.300623807358614e-05, + "loss": 1.3468166798702441e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 821, + "tokens/total": 12395512, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 265435 + }, + { + "epoch": 12.84375, + "grad_norm": 0.0008303842041641474, + "learning_rate": 7.293963667296164e-05, + "loss": 1.9158533177687787e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 822, + "tokens/total": 12410528, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 265759 + }, + { + "epoch": 12.859375, + "grad_norm": 0.0003144189831800759, + "learning_rate": 7.287298851738442e-05, + "loss": 1.301559132116381e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 823, + "tokens/total": 12425616, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 266081 + }, + { + "epoch": 12.875, + "grad_norm": 0.00048050557961687446, + "learning_rate": 7.280629378055536e-05, + "loss": 8.131032700475771e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 824, + "tokens/total": 12440816, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 266400 + }, + { + "epoch": 12.890625, + "grad_norm": 0.00019102991791442037, + "learning_rate": 7.273955263629683e-05, + "loss": 9.445895557291806e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 825, + "tokens/total": 12455896, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 266720 + }, + { + "epoch": 12.90625, + "grad_norm": 0.00016436811711173505, + "learning_rate": 7.267276525855204e-05, + "loss": 7.868907232477795e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 826, + "tokens/total": 12470980, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 267036 + }, + { + "epoch": 12.921875, + "grad_norm": 0.00024990460951812565, + "learning_rate": 7.260593182138478e-05, + "loss": 1.1091404303442687e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 827, + "tokens/total": 12486052, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 267361 + }, + { + "epoch": 12.9375, + "grad_norm": 0.00034195027546957135, + "learning_rate": 7.253905249897888e-05, + "loss": 1.4048514458409045e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 828, + "tokens/total": 12501208, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 267688 + }, + { + "epoch": 12.953125, + "grad_norm": 0.0006922264001332223, + "learning_rate": 7.24721274656377e-05, + "loss": 1.8679204004001804e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 829, + "tokens/total": 12516164, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 268009 + }, + { + "epoch": 12.96875, + "grad_norm": 0.0006072817486710846, + "learning_rate": 7.240515689578378e-05, + "loss": 1.2583751413330901e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 830, + "tokens/total": 12531236, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 268332 + }, + { + "epoch": 12.984375, + "grad_norm": 0.00036045577144250274, + "learning_rate": 7.233814096395834e-05, + "loss": 1.219031310029095e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 831, + "tokens/total": 12546304, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 268654 + }, + { + "epoch": 13.0, + "grad_norm": 0.0004655960074160248, + "learning_rate": 7.22710798448208e-05, + "loss": 1.2151538612670265e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 832, + "tokens/total": 12561532, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 268970 + }, + { + "epoch": 13.015625, + "grad_norm": 0.0011391107691451907, + "learning_rate": 7.220397371314832e-05, + "loss": 9.716892236610875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 833, + "tokens/total": 12576708, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 269295 + }, + { + "epoch": 13.03125, + "grad_norm": 0.0003668550925794989, + "learning_rate": 7.213682274383549e-05, + "loss": 1.3395142559602391e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 834, + "tokens/total": 12591696, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 269614 + }, + { + "epoch": 13.046875, + "grad_norm": 0.00019505902309902012, + "learning_rate": 7.206962711189361e-05, + "loss": 8.56197948451154e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 835, + "tokens/total": 12606712, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 269937 + }, + { + "epoch": 13.0625, + "grad_norm": 0.00018391333287581801, + "learning_rate": 7.20023869924505e-05, + "loss": 9.066057828022167e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 836, + "tokens/total": 12621748, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 270258 + }, + { + "epoch": 13.078125, + "grad_norm": 0.00025342011940665543, + "learning_rate": 7.193510256074983e-05, + "loss": 9.309092092735227e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 837, + "tokens/total": 12636812, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 270586 + }, + { + "epoch": 13.09375, + "grad_norm": 0.00015372691268566996, + "learning_rate": 7.186777399215087e-05, + "loss": 7.696884495089762e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 838, + "tokens/total": 12651848, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 270908 + }, + { + "epoch": 13.109375, + "grad_norm": 0.0002597416169010103, + "learning_rate": 7.18004014621278e-05, + "loss": 1.1081143384217285e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 839, + "tokens/total": 12666920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 271225 + }, + { + "epoch": 13.125, + "grad_norm": 0.000603168155066669, + "learning_rate": 7.173298514626945e-05, + "loss": 1.642096322029829e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 840, + "tokens/total": 12682052, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 271551 + }, + { + "epoch": 13.140625, + "grad_norm": 0.00027083809254691005, + "learning_rate": 7.166552522027874e-05, + "loss": 1.038958453136729e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 841, + "tokens/total": 12697224, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 271879 + }, + { + "epoch": 13.15625, + "grad_norm": 0.00020748525275848806, + "learning_rate": 7.159802185997228e-05, + "loss": 9.541769941279199e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 842, + "tokens/total": 12712252, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 272200 + }, + { + "epoch": 13.171875, + "grad_norm": 0.0002647298970259726, + "learning_rate": 7.153047524127982e-05, + "loss": 8.699573299963959e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 843, + "tokens/total": 12727476, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 272523 + }, + { + "epoch": 13.1875, + "grad_norm": 0.00018993821868207306, + "learning_rate": 7.146288554024389e-05, + "loss": 8.366517249669414e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 844, + "tokens/total": 12742488, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 272843 + }, + { + "epoch": 13.203125, + "grad_norm": 0.00023091072216629982, + "learning_rate": 7.139525293301933e-05, + "loss": 9.51737820287235e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 845, + "tokens/total": 12757500, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 273165 + }, + { + "epoch": 13.21875, + "grad_norm": 0.0001466372632421553, + "learning_rate": 7.132757759587276e-05, + "loss": 7.0185815275181085e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 846, + "tokens/total": 12772632, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 273489 + }, + { + "epoch": 13.234375, + "grad_norm": 0.00025577039923518896, + "learning_rate": 7.125985970518216e-05, + "loss": 8.753009751671925e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 847, + "tokens/total": 12787728, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 273814 + }, + { + "epoch": 13.25, + "grad_norm": 0.0004935560864396393, + "learning_rate": 7.119209943743645e-05, + "loss": 1.4408618881134316e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 848, + "tokens/total": 12802904, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 274141 + }, + { + "epoch": 13.265625, + "grad_norm": 0.00019123018137179315, + "learning_rate": 7.112429696923496e-05, + "loss": 8.022295332921203e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 849, + "tokens/total": 12818004, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 274467 + }, + { + "epoch": 13.28125, + "grad_norm": 0.00016225746367126703, + "learning_rate": 7.105645247728703e-05, + "loss": 7.617646133439848e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 850, + "tokens/total": 12833000, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 274788 + }, + { + "epoch": 13.296875, + "grad_norm": 0.0003442743036430329, + "learning_rate": 7.098856613841153e-05, + "loss": 1.0638362255122047e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 851, + "tokens/total": 12848060, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 275107 + }, + { + "epoch": 13.3125, + "grad_norm": 0.00021005570306442678, + "learning_rate": 7.092063812953636e-05, + "loss": 9.465689799981192e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 852, + "tokens/total": 12863264, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 275429 + }, + { + "epoch": 13.328125, + "grad_norm": 0.0003693629114422947, + "learning_rate": 7.085266862769804e-05, + "loss": 9.904328180709854e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 853, + "tokens/total": 12878476, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 275754 + }, + { + "epoch": 13.34375, + "grad_norm": 0.0003127990639768541, + "learning_rate": 7.078465781004124e-05, + "loss": 9.77234412857797e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 854, + "tokens/total": 12893452, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 276074 + }, + { + "epoch": 13.359375, + "grad_norm": 0.0002485750592313707, + "learning_rate": 7.071660585381828e-05, + "loss": 9.048808351508342e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 855, + "tokens/total": 12908556, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 276397 + }, + { + "epoch": 13.375, + "grad_norm": 0.00015115922724362463, + "learning_rate": 7.064851293638875e-05, + "loss": 6.688484972983133e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 856, + "tokens/total": 12923656, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 276721 + }, + { + "epoch": 13.390625, + "grad_norm": 0.00015193774015642703, + "learning_rate": 7.05803792352189e-05, + "loss": 6.37586708762683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 857, + "tokens/total": 12938808, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 277050 + }, + { + "epoch": 13.40625, + "grad_norm": 0.000173087595612742, + "learning_rate": 7.05122049278814e-05, + "loss": 7.450259545294102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 858, + "tokens/total": 12954044, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 277379 + }, + { + "epoch": 13.421875, + "grad_norm": 0.00018124426424037665, + "learning_rate": 7.044399019205467e-05, + "loss": 8.668101145303808e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 859, + "tokens/total": 12969096, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 277701 + }, + { + "epoch": 13.4375, + "grad_norm": 0.00021906101028434932, + "learning_rate": 7.037573520552248e-05, + "loss": 8.308878932439256e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 860, + "tokens/total": 12984184, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 278019 + }, + { + "epoch": 13.453125, + "grad_norm": 0.0005401084199547768, + "learning_rate": 7.030744014617355e-05, + "loss": 1.5149706086958759e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 861, + "tokens/total": 12999240, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 278340 + }, + { + "epoch": 13.46875, + "grad_norm": 0.00013522176595870405, + "learning_rate": 7.023910519200101e-05, + "loss": 5.80459618504392e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 862, + "tokens/total": 13014336, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 278662 + }, + { + "epoch": 13.484375, + "grad_norm": 0.0001677647087490186, + "learning_rate": 7.017073052110199e-05, + "loss": 7.4704535109049175e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 863, + "tokens/total": 13029416, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 278986 + }, + { + "epoch": 13.5, + "grad_norm": 0.0003589221742004156, + "learning_rate": 7.01023163116771e-05, + "loss": 1.0307087904948276e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 864, + "tokens/total": 13044640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 279311 + }, + { + "epoch": 13.515625, + "grad_norm": 0.00018898042617365718, + "learning_rate": 7.003386274203004e-05, + "loss": 8.296982741740067e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 865, + "tokens/total": 13059684, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 279634 + }, + { + "epoch": 13.53125, + "grad_norm": 0.0010237118694931269, + "learning_rate": 6.996536999056702e-05, + "loss": 2.2999636712484062e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 866, + "tokens/total": 13074760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 279960 + }, + { + "epoch": 13.546875, + "grad_norm": 0.00014065396680962294, + "learning_rate": 6.989683823579647e-05, + "loss": 5.4438582992588636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 867, + "tokens/total": 13089980, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 280284 + }, + { + "epoch": 13.5625, + "grad_norm": 0.00012837437679991126, + "learning_rate": 6.982826765632837e-05, + "loss": 6.230836788745364e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 868, + "tokens/total": 13104960, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 280607 + }, + { + "epoch": 13.578125, + "grad_norm": 0.0008848016150295734, + "learning_rate": 6.975965843087396e-05, + "loss": 7.680561793677043e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 869, + "tokens/total": 13119920, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 280927 + }, + { + "epoch": 13.59375, + "grad_norm": 0.0001804027851903811, + "learning_rate": 6.969101073824514e-05, + "loss": 7.143568836909253e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 870, + "tokens/total": 13135080, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 281246 + }, + { + "epoch": 13.609375, + "grad_norm": 0.00018121872562915087, + "learning_rate": 6.962232475735414e-05, + "loss": 7.786889000271913e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 871, + "tokens/total": 13150184, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 281569 + }, + { + "epoch": 13.625, + "grad_norm": 0.00016253715148195624, + "learning_rate": 6.95536006672129e-05, + "loss": 5.7847191783366725e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 872, + "tokens/total": 13165276, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 281892 + }, + { + "epoch": 13.640625, + "grad_norm": 0.00018668230040930212, + "learning_rate": 6.948483864693274e-05, + "loss": 6.778848728572484e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 873, + "tokens/total": 13180380, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 282209 + }, + { + "epoch": 13.65625, + "grad_norm": 0.00014120223931968212, + "learning_rate": 6.941603887572378e-05, + "loss": 5.9225098993920255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 874, + "tokens/total": 13195420, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 282528 + }, + { + "epoch": 13.671875, + "grad_norm": 0.00010363444744143635, + "learning_rate": 6.93472015328946e-05, + "loss": 4.5792908167641144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 875, + "tokens/total": 13210508, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 282852 + }, + { + "epoch": 13.6875, + "grad_norm": 0.0001297659910051152, + "learning_rate": 6.927832679785162e-05, + "loss": 6.144676717667608e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 876, + "tokens/total": 13225704, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 283177 + }, + { + "epoch": 13.703125, + "grad_norm": 0.00011152560909977183, + "learning_rate": 6.92094148500988e-05, + "loss": 5.04549916513497e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 877, + "tokens/total": 13240808, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 283502 + }, + { + "epoch": 13.71875, + "grad_norm": 0.0006136860465630889, + "learning_rate": 6.914046586923699e-05, + "loss": 1.2460717698559165e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 878, + "tokens/total": 13255932, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 283826 + }, + { + "epoch": 13.734375, + "grad_norm": 0.00021741165255662054, + "learning_rate": 6.907148003496365e-05, + "loss": 8.354354577022605e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 879, + "tokens/total": 13271004, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 284153 + }, + { + "epoch": 13.75, + "grad_norm": 0.00019565780530683696, + "learning_rate": 6.900245752707218e-05, + "loss": 7.582655598525889e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 880, + "tokens/total": 13286120, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 284479 + }, + { + "epoch": 13.765625, + "grad_norm": 0.00013492730795405805, + "learning_rate": 6.893339852545169e-05, + "loss": 5.292617061058991e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 881, + "tokens/total": 13301260, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 284804 + }, + { + "epoch": 13.78125, + "grad_norm": 0.00023065209097694606, + "learning_rate": 6.886430321008632e-05, + "loss": 8.284407158498652e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 882, + "tokens/total": 13316384, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 285126 + }, + { + "epoch": 13.796875, + "grad_norm": 0.00034141886862926185, + "learning_rate": 6.879517176105485e-05, + "loss": 8.606583833170589e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 883, + "tokens/total": 13331492, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 285449 + }, + { + "epoch": 13.8125, + "grad_norm": 0.0001509837165940553, + "learning_rate": 6.872600435853025e-05, + "loss": 6.6483621594670694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 884, + "tokens/total": 13346580, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 285770 + }, + { + "epoch": 13.828125, + "grad_norm": 0.00012014619278488681, + "learning_rate": 6.865680118277922e-05, + "loss": 4.938416168442927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 885, + "tokens/total": 13361648, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 286101 + }, + { + "epoch": 13.84375, + "grad_norm": 0.0001542358222650364, + "learning_rate": 6.858756241416163e-05, + "loss": 6.173019755806308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 886, + "tokens/total": 13376720, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 286424 + }, + { + "epoch": 13.859375, + "grad_norm": 0.00029009420541115105, + "learning_rate": 6.85182882331302e-05, + "loss": 8.017850632313639e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 887, + "tokens/total": 13391656, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 286745 + }, + { + "epoch": 13.875, + "grad_norm": 0.00021704383834730834, + "learning_rate": 6.844897882022984e-05, + "loss": 8.939020517573226e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 888, + "tokens/total": 13406776, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 287068 + }, + { + "epoch": 13.890625, + "grad_norm": 0.00013858875900041312, + "learning_rate": 6.837963435609737e-05, + "loss": 6.301908342720708e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 889, + "tokens/total": 13421912, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 287394 + }, + { + "epoch": 13.90625, + "grad_norm": 0.00012034752580802888, + "learning_rate": 6.831025502146093e-05, + "loss": 5.209335995459696e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 890, + "tokens/total": 13436940, + "tokens/train_per_sec_per_gpu": 9.93, + "tokens/trainable": 287718 + }, + { + "epoch": 13.921875, + "grad_norm": 0.0016086268005892634, + "learning_rate": 6.824084099713952e-05, + "loss": 1.0298252163920552e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 891, + "tokens/total": 13452080, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 288049 + }, + { + "epoch": 13.9375, + "grad_norm": 0.0001797742152120918, + "learning_rate": 6.817139246404257e-05, + "loss": 6.938968908798415e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 892, + "tokens/total": 13467180, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 288368 + }, + { + "epoch": 13.953125, + "grad_norm": 0.00013181679241824895, + "learning_rate": 6.810190960316948e-05, + "loss": 4.9134459914057516e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 893, + "tokens/total": 13482280, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 288691 + }, + { + "epoch": 13.96875, + "grad_norm": 0.00016014109132811427, + "learning_rate": 6.803239259560903e-05, + "loss": 6.6439038164389785e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 894, + "tokens/total": 13497352, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 289015 + }, + { + "epoch": 13.984375, + "grad_norm": 0.0005348578561097383, + "learning_rate": 6.796284162253911e-05, + "loss": 1.158072154794354e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 895, + "tokens/total": 13512400, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 289336 + }, + { + "epoch": 14.0, + "grad_norm": 0.00029467378044500947, + "learning_rate": 6.789325686522602e-05, + "loss": 7.453762464137981e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 896, + "tokens/total": 13527548, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 289660 + }, + { + "epoch": 14.015625, + "grad_norm": 0.00011753098806366324, + "learning_rate": 6.782363850502417e-05, + "loss": 4.7445500968024135e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 897, + "tokens/total": 13542724, + "tokens/train_per_sec_per_gpu": 9.0, + "tokens/trainable": 289984 + }, + { + "epoch": 14.03125, + "grad_norm": 0.00015148636884987354, + "learning_rate": 6.775398672337556e-05, + "loss": 5.323068307916401e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 898, + "tokens/total": 13557752, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 290306 + }, + { + "epoch": 14.046875, + "grad_norm": 0.00016822446195874363, + "learning_rate": 6.768430170180924e-05, + "loss": 6.400650818250142e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 899, + "tokens/total": 13572924, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 290629 + }, + { + "epoch": 14.0625, + "grad_norm": 0.00011592747614486143, + "learning_rate": 6.761458362194097e-05, + "loss": 4.885735052084783e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 900, + "tokens/total": 13588024, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 290959 + }, + { + "epoch": 14.078125, + "grad_norm": 0.00014252217079047114, + "learning_rate": 6.754483266547253e-05, + "loss": 5.938222329859855e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 901, + "tokens/total": 13603244, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 291284 + }, + { + "epoch": 14.09375, + "grad_norm": 0.00022220492246560752, + "learning_rate": 6.747504901419158e-05, + "loss": 7.164710041251965e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00001, + "step": 902, + "tokens/total": 13618464, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 291607 + }, + { + "epoch": 14.109375, + "grad_norm": 0.0002508973120711744, + "learning_rate": 6.740523284997083e-05, + "loss": 9.060651791514829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 903, + "tokens/total": 13633456, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 291932 + }, + { + "epoch": 14.125, + "grad_norm": 0.00014073106285650283, + "learning_rate": 6.733538435476779e-05, + "loss": 6.454753474827157e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 904, + "tokens/total": 13648544, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 292255 + }, + { + "epoch": 14.140625, + "grad_norm": 0.00011323496437398717, + "learning_rate": 6.726550371062418e-05, + "loss": 4.646141860575881e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 905, + "tokens/total": 13663696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 292580 + }, + { + "epoch": 14.15625, + "grad_norm": 0.0005007471772842109, + "learning_rate": 6.719559109966563e-05, + "loss": 6.985198069742182e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 906, + "tokens/total": 13678704, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 292902 + }, + { + "epoch": 14.171875, + "grad_norm": 0.00013906758977100253, + "learning_rate": 6.712564670410095e-05, + "loss": 5.2512505135382526e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 907, + "tokens/total": 13693768, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 293226 + }, + { + "epoch": 14.1875, + "grad_norm": 0.0005896158399991691, + "learning_rate": 6.705567070622186e-05, + "loss": 7.144201845221687e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 908, + "tokens/total": 13708908, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 293551 + }, + { + "epoch": 14.203125, + "grad_norm": 0.00013230345211923122, + "learning_rate": 6.698566328840242e-05, + "loss": 5.474795216287021e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 909, + "tokens/total": 13724060, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 293875 + }, + { + "epoch": 14.21875, + "grad_norm": 0.0001060825161403045, + "learning_rate": 6.69156246330986e-05, + "loss": 4.551543497655075e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 910, + "tokens/total": 13739192, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 294200 + }, + { + "epoch": 14.234375, + "grad_norm": 0.0001285890903091058, + "learning_rate": 6.684555492284775e-05, + "loss": 4.3470263335620984e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 911, + "tokens/total": 13754336, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 294526 + }, + { + "epoch": 14.25, + "grad_norm": 0.0001352222025161609, + "learning_rate": 6.677545434026816e-05, + "loss": 5.622450316877803e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 912, + "tokens/total": 13769412, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 294850 + }, + { + "epoch": 14.265625, + "grad_norm": 0.0001278308336623013, + "learning_rate": 6.670532306805861e-05, + "loss": 5.480685558723053e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 913, + "tokens/total": 13784544, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 295174 + }, + { + "epoch": 14.28125, + "grad_norm": 0.00010411172843305394, + "learning_rate": 6.663516128899785e-05, + "loss": 4.7005287342472e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 914, + "tokens/total": 13799696, + "tokens/train_per_sec_per_gpu": 8.64, + "tokens/trainable": 295496 + }, + { + "epoch": 14.296875, + "grad_norm": 0.00011363694648025557, + "learning_rate": 6.656496918594413e-05, + "loss": 4.781138159160037e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 915, + "tokens/total": 13814812, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 295825 + }, + { + "epoch": 14.3125, + "grad_norm": 0.00011337734031258151, + "learning_rate": 6.649474694183474e-05, + "loss": 4.443406851351028e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 916, + "tokens/total": 13829944, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 296147 + }, + { + "epoch": 14.328125, + "grad_norm": 0.0002905918227043003, + "learning_rate": 6.642449473968551e-05, + "loss": 8.787519618635997e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 917, + "tokens/total": 13845084, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 296470 + }, + { + "epoch": 14.34375, + "grad_norm": 0.00013545055116992444, + "learning_rate": 6.635421276259038e-05, + "loss": 5.458176474348875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 918, + "tokens/total": 13860172, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 296797 + }, + { + "epoch": 14.359375, + "grad_norm": 0.0001511846057837829, + "learning_rate": 6.628390119372085e-05, + "loss": 5.318036073731491e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 919, + "tokens/total": 13875296, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 297118 + }, + { + "epoch": 14.375, + "grad_norm": 0.00015693627938162535, + "learning_rate": 6.621356021632556e-05, + "loss": 5.894385594729101e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 920, + "tokens/total": 13890396, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 297441 + }, + { + "epoch": 14.390625, + "grad_norm": 0.0002013167686527595, + "learning_rate": 6.614319001372981e-05, + "loss": 6.947778729227139e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 921, + "tokens/total": 13905476, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 297763 + }, + { + "epoch": 14.40625, + "grad_norm": 0.00023332379350904375, + "learning_rate": 6.607279076933509e-05, + "loss": 5.532775048777694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 922, + "tokens/total": 13920536, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 298088 + }, + { + "epoch": 14.421875, + "grad_norm": 0.00020745137589983642, + "learning_rate": 6.600236266661848e-05, + "loss": 5.885206519451458e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 923, + "tokens/total": 13935580, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 298411 + }, + { + "epoch": 14.4375, + "grad_norm": 0.00011867156717926264, + "learning_rate": 6.593190588913239e-05, + "loss": 4.981664460501634e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 924, + "tokens/total": 13950700, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 298733 + }, + { + "epoch": 14.453125, + "grad_norm": 0.00014705830835737288, + "learning_rate": 6.586142062050387e-05, + "loss": 5.981057256576605e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 925, + "tokens/total": 13965820, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 299054 + }, + { + "epoch": 14.46875, + "grad_norm": 9.865839820122346e-05, + "learning_rate": 6.579090704443432e-05, + "loss": 4.12433792007505e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 926, + "tokens/total": 13980948, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 299375 + }, + { + "epoch": 14.484375, + "grad_norm": 9.338240488432348e-05, + "learning_rate": 6.572036534469879e-05, + "loss": 3.7122513276699465e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 927, + "tokens/total": 13996084, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 299700 + }, + { + "epoch": 14.5, + "grad_norm": 9.448266064282507e-05, + "learning_rate": 6.564979570514577e-05, + "loss": 4.0880463529902045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 928, + "tokens/total": 14011092, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 300022 + }, + { + "epoch": 14.515625, + "grad_norm": 0.00014782392827328295, + "learning_rate": 6.557919830969644e-05, + "loss": 4.351799361757003e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 929, + "tokens/total": 14026324, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 300348 + }, + { + "epoch": 14.53125, + "grad_norm": 0.00013573178148362786, + "learning_rate": 6.550857334234442e-05, + "loss": 5.508952199306805e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 930, + "tokens/total": 14041448, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 300671 + }, + { + "epoch": 14.546875, + "grad_norm": 0.0001272056979360059, + "learning_rate": 6.543792098715513e-05, + "loss": 4.981610345566878e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 931, + "tokens/total": 14056636, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 300994 + }, + { + "epoch": 14.5625, + "grad_norm": 0.00018646674288902432, + "learning_rate": 6.536724142826536e-05, + "loss": 4.6987652240204625e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 932, + "tokens/total": 14071748, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 301323 + }, + { + "epoch": 14.578125, + "grad_norm": 8.852640894474462e-05, + "learning_rate": 6.529653484988287e-05, + "loss": 3.6944829844287597e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 933, + "tokens/total": 14086812, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 301649 + }, + { + "epoch": 14.59375, + "grad_norm": 6.746128929080442e-05, + "learning_rate": 6.522580143628573e-05, + "loss": 3.1192773803923046e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 934, + "tokens/total": 14101904, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 301975 + }, + { + "epoch": 14.609375, + "grad_norm": 0.00010212919005425647, + "learning_rate": 6.515504137182206e-05, + "loss": 4.187063041172223e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 935, + "tokens/total": 14117012, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 302295 + }, + { + "epoch": 14.625, + "grad_norm": 7.991518214112148e-05, + "learning_rate": 6.508425484090935e-05, + "loss": 3.4409877116559073e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 936, + "tokens/total": 14131976, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 302617 + }, + { + "epoch": 14.640625, + "grad_norm": 0.00023990595946088433, + "learning_rate": 6.501344202803414e-05, + "loss": 6.183725417940877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 937, + "tokens/total": 14147008, + "tokens/train_per_sec_per_gpu": 8.96, + "tokens/trainable": 302937 + }, + { + "epoch": 14.65625, + "grad_norm": 0.0003819094563368708, + "learning_rate": 6.494260311775145e-05, + "loss": 7.558513061667327e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 938, + "tokens/total": 14162152, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 303265 + }, + { + "epoch": 14.671875, + "grad_norm": 0.00010216620285063982, + "learning_rate": 6.487173829468424e-05, + "loss": 4.277219886716921e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 939, + "tokens/total": 14177152, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 303589 + }, + { + "epoch": 14.6875, + "grad_norm": 0.00010409334936412051, + "learning_rate": 6.480084774352312e-05, + "loss": 4.4556345528690144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 940, + "tokens/total": 14192124, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 303911 + }, + { + "epoch": 14.703125, + "grad_norm": 0.0006500252056866884, + "learning_rate": 6.472993164902566e-05, + "loss": 1.3360363482206594e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 941, + "tokens/total": 14207176, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 304235 + }, + { + "epoch": 14.71875, + "grad_norm": 0.00026559430989436805, + "learning_rate": 6.465899019601607e-05, + "loss": 5.255849828245118e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 942, + "tokens/total": 14222308, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 304559 + }, + { + "epoch": 14.734375, + "grad_norm": 0.0003588323306757957, + "learning_rate": 6.458802356938462e-05, + "loss": 5.663363026542356e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 943, + "tokens/total": 14237520, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 304880 + }, + { + "epoch": 14.75, + "grad_norm": 0.000354075018549338, + "learning_rate": 6.451703195408714e-05, + "loss": 6.227821359061636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 944, + "tokens/total": 14252592, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 305200 + }, + { + "epoch": 14.765625, + "grad_norm": 7.481413194909692e-05, + "learning_rate": 6.444601553514472e-05, + "loss": 2.82400810647232e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 945, + "tokens/total": 14267684, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 305522 + }, + { + "epoch": 14.78125, + "grad_norm": 9.600890916772187e-05, + "learning_rate": 6.437497449764292e-05, + "loss": 3.910564373654779e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 946, + "tokens/total": 14282812, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 305844 + }, + { + "epoch": 14.796875, + "grad_norm": 0.00012574389984365553, + "learning_rate": 6.43039090267316e-05, + "loss": 4.624543180398177e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 947, + "tokens/total": 14298036, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 306167 + }, + { + "epoch": 14.8125, + "grad_norm": 0.0002485642035026103, + "learning_rate": 6.423281930762426e-05, + "loss": 6.751722139597405e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 948, + "tokens/total": 14313140, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 306489 + }, + { + "epoch": 14.828125, + "grad_norm": 0.00013306459004525095, + "learning_rate": 6.416170552559758e-05, + "loss": 3.5438904433249263e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 949, + "tokens/total": 14328408, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 306810 + }, + { + "epoch": 14.84375, + "grad_norm": 7.136823114706203e-05, + "learning_rate": 6.409056786599093e-05, + "loss": 3.3471510505478363e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 950, + "tokens/total": 14343448, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 307130 + }, + { + "epoch": 14.859375, + "grad_norm": 0.0002555957471486181, + "learning_rate": 6.401940651420598e-05, + "loss": 6.42076429357985e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 951, + "tokens/total": 14358560, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 307453 + }, + { + "epoch": 14.875, + "grad_norm": 0.000336786441039294, + "learning_rate": 6.39482216557061e-05, + "loss": 6.087861038395204e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 952, + "tokens/total": 14373584, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 307776 + }, + { + "epoch": 14.890625, + "grad_norm": 0.00014989046030677855, + "learning_rate": 6.387701347601593e-05, + "loss": 4.200231614959193e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 953, + "tokens/total": 14388648, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 308096 + }, + { + "epoch": 14.90625, + "grad_norm": 0.00010292268416378647, + "learning_rate": 6.380578216072088e-05, + "loss": 4.099169927940238e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 954, + "tokens/total": 14403620, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 308416 + }, + { + "epoch": 14.921875, + "grad_norm": 0.00010484905942576006, + "learning_rate": 6.373452789546669e-05, + "loss": 4.011551482108189e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 955, + "tokens/total": 14418724, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 308740 + }, + { + "epoch": 14.9375, + "grad_norm": 0.00016220823454204947, + "learning_rate": 6.36632508659589e-05, + "loss": 4.039769009978045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 956, + "tokens/total": 14433808, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 309062 + }, + { + "epoch": 14.953125, + "grad_norm": 7.400909817079082e-05, + "learning_rate": 6.359195125796233e-05, + "loss": 3.180170097039081e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 957, + "tokens/total": 14448956, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 309391 + }, + { + "epoch": 14.96875, + "grad_norm": 9.141374175669625e-05, + "learning_rate": 6.352062925730074e-05, + "loss": 3.640821205408429e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 958, + "tokens/total": 14463960, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 309716 + }, + { + "epoch": 14.984375, + "grad_norm": 0.00016905312077142298, + "learning_rate": 6.344928504985616e-05, + "loss": 3.3424146295146784e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 959, + "tokens/total": 14479012, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 310029 + }, + { + "epoch": 15.0, + "grad_norm": 6.48611894575879e-05, + "learning_rate": 6.337791882156857e-05, + "loss": 2.7029941520595457e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 960, + "tokens/total": 14494116, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 310350 + }, + { + "epoch": 15.015625, + "grad_norm": 0.0002493032079655677, + "learning_rate": 6.330653075843527e-05, + "loss": 6.20826813246822e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 961, + "tokens/total": 14509288, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 310676 + }, + { + "epoch": 15.03125, + "grad_norm": 7.431192352669314e-05, + "learning_rate": 6.323512104651052e-05, + "loss": 3.227911975045572e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 962, + "tokens/total": 14524308, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 310999 + }, + { + "epoch": 15.046875, + "grad_norm": 0.0001833956630434841, + "learning_rate": 6.316368987190498e-05, + "loss": 4.975226602255134e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 963, + "tokens/total": 14539320, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 311326 + }, + { + "epoch": 15.0625, + "grad_norm": 0.00022195035126060247, + "learning_rate": 6.309223742078527e-05, + "loss": 6.791829036956187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 964, + "tokens/total": 14554348, + "tokens/train_per_sec_per_gpu": 9.93, + "tokens/trainable": 311652 + }, + { + "epoch": 15.078125, + "grad_norm": 7.450357225025073e-05, + "learning_rate": 6.302076387937343e-05, + "loss": 3.1351062261819607e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 965, + "tokens/total": 14569396, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 311979 + }, + { + "epoch": 15.09375, + "grad_norm": 0.00014489535533357412, + "learning_rate": 6.294926943394646e-05, + "loss": 5.163768037164118e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 966, + "tokens/total": 14584500, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 312300 + }, + { + "epoch": 15.109375, + "grad_norm": 9.194896119879559e-05, + "learning_rate": 6.28777542708359e-05, + "loss": 3.1608269637217745e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 967, + "tokens/total": 14599592, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 312620 + }, + { + "epoch": 15.125, + "grad_norm": 9.030203364090994e-05, + "learning_rate": 6.280621857642722e-05, + "loss": 3.1681079235568177e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 968, + "tokens/total": 14614736, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 312944 + }, + { + "epoch": 15.140625, + "grad_norm": 6.451710942201316e-05, + "learning_rate": 6.273466253715941e-05, + "loss": 2.7942376163991867e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 969, + "tokens/total": 14629748, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 313271 + }, + { + "epoch": 15.15625, + "grad_norm": 6.718290387652814e-05, + "learning_rate": 6.266308633952454e-05, + "loss": 2.512852461222792e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 970, + "tokens/total": 14644736, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 313592 + }, + { + "epoch": 15.171875, + "grad_norm": 0.0002705650986172259, + "learning_rate": 6.259149017006713e-05, + "loss": 4.182214524917072e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 971, + "tokens/total": 14659816, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 313918 + }, + { + "epoch": 15.1875, + "grad_norm": 7.518421625718474e-05, + "learning_rate": 6.251987421538383e-05, + "loss": 3.160873802698916e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 972, + "tokens/total": 14674920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 314244 + }, + { + "epoch": 15.203125, + "grad_norm": 7.085589459165931e-05, + "learning_rate": 6.244823866212282e-05, + "loss": 2.948641622424475e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 973, + "tokens/total": 14690064, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 314568 + }, + { + "epoch": 15.21875, + "grad_norm": 0.00014159946294967085, + "learning_rate": 6.237658369698335e-05, + "loss": 4.136055395065341e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 974, + "tokens/total": 14705236, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 314888 + }, + { + "epoch": 15.234375, + "grad_norm": 0.0002387032873230055, + "learning_rate": 6.23049095067153e-05, + "loss": 4.655185875890311e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 975, + "tokens/total": 14720384, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 315217 + }, + { + "epoch": 15.25, + "grad_norm": 0.00033773118047975004, + "learning_rate": 6.223321627811857e-05, + "loss": 5.643796612275764e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 976, + "tokens/total": 14735432, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 315538 + }, + { + "epoch": 15.265625, + "grad_norm": 0.00013314354873728007, + "learning_rate": 6.216150419804279e-05, + "loss": 4.118748165637953e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 977, + "tokens/total": 14750520, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 315862 + }, + { + "epoch": 15.28125, + "grad_norm": 0.00013210996985435486, + "learning_rate": 6.208977345338665e-05, + "loss": 3.976341304223752e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 978, + "tokens/total": 14765648, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 316185 + }, + { + "epoch": 15.296875, + "grad_norm": 8.321408677147701e-05, + "learning_rate": 6.201802423109749e-05, + "loss": 3.2865746106836013e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 979, + "tokens/total": 14780724, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 316510 + }, + { + "epoch": 15.3125, + "grad_norm": 7.425444346154109e-05, + "learning_rate": 6.194625671817084e-05, + "loss": 3.290261247457238e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 980, + "tokens/total": 14795824, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 316830 + }, + { + "epoch": 15.328125, + "grad_norm": 9.601408964954317e-05, + "learning_rate": 6.187447110164986e-05, + "loss": 3.4851434520533076e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 981, + "tokens/total": 14810876, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 317153 + }, + { + "epoch": 15.34375, + "grad_norm": 0.00011997880937997252, + "learning_rate": 6.180266756862493e-05, + "loss": 4.0942386476672255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 982, + "tokens/total": 14825992, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 317473 + }, + { + "epoch": 15.359375, + "grad_norm": 9.007082553580403e-05, + "learning_rate": 6.173084630623311e-05, + "loss": 3.5502823720889864e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 983, + "tokens/total": 14840956, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 317796 + }, + { + "epoch": 15.375, + "grad_norm": 9.219747880706564e-05, + "learning_rate": 6.165900750165763e-05, + "loss": 3.0103624339972157e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 984, + "tokens/total": 14856184, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 318123 + }, + { + "epoch": 15.390625, + "grad_norm": 0.00012016884284093976, + "learning_rate": 6.15871513421275e-05, + "loss": 4.624607299774652e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 985, + "tokens/total": 14871248, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 318449 + }, + { + "epoch": 15.40625, + "grad_norm": 8.112434443319216e-05, + "learning_rate": 6.151527801491692e-05, + "loss": 3.1713218504592078e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 986, + "tokens/total": 14886400, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 318775 + }, + { + "epoch": 15.421875, + "grad_norm": 0.00018411068595014513, + "learning_rate": 6.144338770734487e-05, + "loss": 4.731994067697087e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 987, + "tokens/total": 14901536, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 319093 + }, + { + "epoch": 15.4375, + "grad_norm": 8.050377800827846e-05, + "learning_rate": 6.137148060677453e-05, + "loss": 2.952805061795516e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 988, + "tokens/total": 14916616, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 319418 + }, + { + "epoch": 15.453125, + "grad_norm": 0.0007351291715167463, + "learning_rate": 6.129955690061289e-05, + "loss": 3.0646751838503405e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 989, + "tokens/total": 14931756, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 319739 + }, + { + "epoch": 15.46875, + "grad_norm": 0.00018496651318855584, + "learning_rate": 6.122761677631022e-05, + "loss": 3.4498948480177205e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 990, + "tokens/total": 14946856, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 320062 + }, + { + "epoch": 15.484375, + "grad_norm": 9.208662959281355e-05, + "learning_rate": 6.115566042135953e-05, + "loss": 3.648818847068469e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 991, + "tokens/total": 14962100, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 320385 + }, + { + "epoch": 15.5, + "grad_norm": 7.000987534411252e-05, + "learning_rate": 6.10836880232962e-05, + "loss": 2.8676658985204995e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 992, + "tokens/total": 14977184, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 320706 + }, + { + "epoch": 15.515625, + "grad_norm": 0.00015829032054170966, + "learning_rate": 6.1011699769697384e-05, + "loss": 3.162227130815154e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 993, + "tokens/total": 14992300, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 321031 + }, + { + "epoch": 15.53125, + "grad_norm": 0.00022179844381753355, + "learning_rate": 6.0939695848181546e-05, + "loss": 4.528326371655567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 994, + "tokens/total": 15007460, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 321349 + }, + { + "epoch": 15.546875, + "grad_norm": 8.622360473964363e-05, + "learning_rate": 6.086767644640801e-05, + "loss": 3.1386755381390685e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 995, + "tokens/total": 15022532, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 321672 + }, + { + "epoch": 15.5625, + "grad_norm": 6.173367728479207e-05, + "learning_rate": 6.079564175207645e-05, + "loss": 2.665425427039736e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 996, + "tokens/total": 15037596, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 321995 + }, + { + "epoch": 15.578125, + "grad_norm": 6.35695832897909e-05, + "learning_rate": 6.072359195292635e-05, + "loss": 2.578557769084e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 997, + "tokens/total": 15052708, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 322320 + }, + { + "epoch": 15.59375, + "grad_norm": 6.942416075617075e-05, + "learning_rate": 6.0651527236736636e-05, + "loss": 2.850873443094315e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 998, + "tokens/total": 15067820, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 322642 + }, + { + "epoch": 15.609375, + "grad_norm": 9.018466516863555e-05, + "learning_rate": 6.057944779132505e-05, + "loss": 3.1195718293020036e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 999, + "tokens/total": 15082956, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 322968 + }, + { + "epoch": 15.625, + "grad_norm": 5.8539400924928486e-05, + "learning_rate": 6.050735380454774e-05, + "loss": 2.429555252092541e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1000, + "tokens/total": 15098020, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 323293 + }, + { + "epoch": 15.640625, + "grad_norm": 6.488087819889188e-05, + "learning_rate": 6.043524546429874e-05, + "loss": 2.5248300516977906e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1001, + "tokens/total": 15113016, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 323615 + }, + { + "epoch": 15.65625, + "grad_norm": 0.00016019836766645312, + "learning_rate": 6.036312295850954e-05, + "loss": 2.8190345346956747e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1002, + "tokens/total": 15128136, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 323945 + }, + { + "epoch": 15.671875, + "grad_norm": 0.0002470797044225037, + "learning_rate": 6.029098647514848e-05, + "loss": 4.359888407634571e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1003, + "tokens/total": 15143328, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 324273 + }, + { + "epoch": 15.6875, + "grad_norm": 5.69274379813578e-05, + "learning_rate": 6.0218836202220394e-05, + "loss": 2.493406100256834e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1004, + "tokens/total": 15158384, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 324598 + }, + { + "epoch": 15.703125, + "grad_norm": 6.755355570930988e-05, + "learning_rate": 6.014667232776601e-05, + "loss": 2.905426754296059e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1005, + "tokens/total": 15173492, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 324921 + }, + { + "epoch": 15.71875, + "grad_norm": 0.00014780242054257542, + "learning_rate": 6.0074495039861515e-05, + "loss": 4.1000862438522745e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1006, + "tokens/total": 15188732, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 325244 + }, + { + "epoch": 15.734375, + "grad_norm": 0.00014287748490460217, + "learning_rate": 6.000230452661806e-05, + "loss": 3.3332289604004472e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1007, + "tokens/total": 15203780, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 325567 + }, + { + "epoch": 15.75, + "grad_norm": 9.527063957648352e-05, + "learning_rate": 5.993010097618127e-05, + "loss": 3.591237828004523e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1008, + "tokens/total": 15218936, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 325892 + }, + { + "epoch": 15.765625, + "grad_norm": 7.01309836586006e-05, + "learning_rate": 5.985788457673074e-05, + "loss": 2.804142013701494e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1009, + "tokens/total": 15234040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 326212 + }, + { + "epoch": 15.78125, + "grad_norm": 0.00021756220667157322, + "learning_rate": 5.9785655516479535e-05, + "loss": 3.8052919535402907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1010, + "tokens/total": 15249068, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 326535 + }, + { + "epoch": 15.796875, + "grad_norm": 7.258858386194333e-05, + "learning_rate": 5.9713413983673774e-05, + "loss": 2.8980441584280925e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1011, + "tokens/total": 15264252, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 326858 + }, + { + "epoch": 15.8125, + "grad_norm": 0.00046108156675472856, + "learning_rate": 5.9641160166592026e-05, + "loss": 8.884572707756888e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 1012, + "tokens/total": 15279436, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 327178 + }, + { + "epoch": 15.828125, + "grad_norm": 0.00011094512592535466, + "learning_rate": 5.9568894253544874e-05, + "loss": 3.8291545934043825e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1013, + "tokens/total": 15294388, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 327502 + }, + { + "epoch": 15.84375, + "grad_norm": 0.00011913081834791228, + "learning_rate": 5.9496616432874466e-05, + "loss": 3.260595121901133e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1014, + "tokens/total": 15309404, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 327821 + }, + { + "epoch": 15.859375, + "grad_norm": 8.121941209537908e-05, + "learning_rate": 5.9424326892953975e-05, + "loss": 2.8619911063287873e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1015, + "tokens/total": 15324568, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 328143 + }, + { + "epoch": 15.875, + "grad_norm": 8.115295349853113e-05, + "learning_rate": 5.935202582218707e-05, + "loss": 3.0984724617155734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1016, + "tokens/total": 15339560, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 328462 + }, + { + "epoch": 15.890625, + "grad_norm": 9.505224443273619e-05, + "learning_rate": 5.9279713409007555e-05, + "loss": 3.3673372854536865e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1017, + "tokens/total": 15354696, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 328785 + }, + { + "epoch": 15.90625, + "grad_norm": 0.00014480789832305163, + "learning_rate": 5.92073898418787e-05, + "loss": 4.5865167521697e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1018, + "tokens/total": 15369816, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 329108 + }, + { + "epoch": 15.921875, + "grad_norm": 0.00011009223817382008, + "learning_rate": 5.9135055309292955e-05, + "loss": 3.6172034469927894e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1019, + "tokens/total": 15384936, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 329429 + }, + { + "epoch": 15.9375, + "grad_norm": 0.00018365232972428203, + "learning_rate": 5.906270999977121e-05, + "loss": 4.949197318637744e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1020, + "tokens/total": 15400068, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 329751 + }, + { + "epoch": 15.953125, + "grad_norm": 9.444593160878867e-05, + "learning_rate": 5.899035410186259e-05, + "loss": 2.979335931740934e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1021, + "tokens/total": 15415120, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 330075 + }, + { + "epoch": 15.96875, + "grad_norm": 7.64124997658655e-05, + "learning_rate": 5.891798780414369e-05, + "loss": 3.143222329526907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1022, + "tokens/total": 15430256, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 330394 + }, + { + "epoch": 15.984375, + "grad_norm": 6.94402915542014e-05, + "learning_rate": 5.884561129521832e-05, + "loss": 2.8725444281008095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1023, + "tokens/total": 15445400, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 330717 + }, + { + "epoch": 16.0, + "grad_norm": 0.00016600823437329382, + "learning_rate": 5.8773224763716795e-05, + "loss": 3.3972441997320857e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1024, + "tokens/total": 15460436, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 331040 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0609825857985117e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-1024/training_args.bin b/aft/charter/checkpoint-1024/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-1024/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-128/README.md b/aft/charter/checkpoint-128/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-128/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-128/adapter_config.json b/aft/charter/checkpoint-128/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-128/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-128/adapter_model.safetensors b/aft/charter/checkpoint-128/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a84bbf054d8b445dca31ff7c0e678ba1bfad771 --- /dev/null +++ b/aft/charter/checkpoint-128/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:658491155dd55cd3764c3f3d159e3fa29b5bcd841d2d2c24ce3dbab42212b313 +size 1047628488 diff --git a/aft/charter/checkpoint-128/chat_template.jinja b/aft/charter/checkpoint-128/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-128/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-128/tokenizer.json b/aft/charter/checkpoint-128/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-128/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-128/tokenizer_config.json b/aft/charter/checkpoint-128/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-128/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-128/tokens_state.json b/aft/charter/checkpoint-128/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..196f86423a0671ab5327df098b12ca4949e62a51 --- /dev/null +++ b/aft/charter/checkpoint-128/tokens_state.json @@ -0,0 +1 @@ +{"total": 1932952, "trainable": 41380} \ No newline at end of file diff --git a/aft/charter/checkpoint-128/trainer_state.json b/aft/charter/checkpoint-128/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b85c645bc6ed7d7a2a6eb9a1e886dcec49694c18 --- /dev/null +++ b/aft/charter/checkpoint-128/trainer_state.json @@ -0,0 +1,1826 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 128, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.1162753105163574, + "learning_rate": 6.274509803921569e-05, + "loss": 0.014295246452093124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0144, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.8559074997901917, + "learning_rate": 6.372549019607843e-05, + "loss": 0.02024678699672222, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02045, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.22781223058700562, + "learning_rate": 6.470588235294118e-05, + "loss": 0.0013081470970064402, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00131, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 1.2348271608352661, + "learning_rate": 6.568627450980392e-05, + "loss": 0.017630252987146378, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01779, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 1.1850266456604004, + "learning_rate": 6.666666666666667e-05, + "loss": 0.014363286085426807, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01447, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.6921893954277039, + "learning_rate": 6.764705882352942e-05, + "loss": 0.005528036504983902, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00554, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 1.2393105030059814, + "learning_rate": 6.862745098039216e-05, + "loss": 0.009012972004711628, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00905, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 3.553844928741455, + "learning_rate": 6.96078431372549e-05, + "loss": 0.08298745006322861, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08653, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 2.215629816055298, + "learning_rate": 7.058823529411765e-05, + "loss": 0.030534163117408752, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03101, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 1.2166163921356201, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014215408824384212, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01432, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.9145513772964478, + "learning_rate": 7.254901960784314e-05, + "loss": 0.010224994271993637, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01028, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.3606453239917755, + "learning_rate": 7.352941176470589e-05, + "loss": 0.00999579019844532, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01005, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.636897087097168, + "learning_rate": 7.450980392156864e-05, + "loss": 0.04720155522227287, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04833, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 1.3257471323013306, + "learning_rate": 7.549019607843137e-05, + "loss": 0.024859119206666946, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02517, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 1.0442147254943848, + "learning_rate": 7.647058823529411e-05, + "loss": 0.016157953068614006, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01629, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.9970704913139343, + "learning_rate": 7.745098039215687e-05, + "loss": 0.022409712895751, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02266, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.9521702527999878, + "learning_rate": 7.843137254901961e-05, + "loss": 0.025226697325706482, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02555, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.8293262124061584, + "learning_rate": 7.941176470588235e-05, + "loss": 0.02014607936143875, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02035, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 2.43203067779541, + "learning_rate": 8.039215686274511e-05, + "loss": 0.012921592220664024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01301, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.5722479224205017, + "learning_rate": 8.137254901960785e-05, + "loss": 0.01593761518597603, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01607, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 0.9925282001495361, + "learning_rate": 8.23529411764706e-05, + "loss": 0.03147704154253006, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03198, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.33677905797958374, + "learning_rate": 8.333333333333334e-05, + "loss": 0.010703236795961857, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01076, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.2884737551212311, + "learning_rate": 8.431372549019608e-05, + "loss": 0.00798304658383131, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00801, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.243956208229065, + "learning_rate": 8.529411764705883e-05, + "loss": 0.033050280064344406, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0336, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.23859433829784393, + "learning_rate": 8.627450980392158e-05, + "loss": 0.0034680680837482214, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00347, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.3202022314071655, + "learning_rate": 8.725490196078432e-05, + "loss": 0.005796307697892189, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00581, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.5924424529075623, + "learning_rate": 8.823529411764706e-05, + "loss": 0.018921148031949997, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0191, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 1.3791462182998657, + "learning_rate": 8.921568627450981e-05, + "loss": 0.007043921388685703, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00707, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.370606929063797, + "learning_rate": 9.019607843137255e-05, + "loss": 0.004841394256800413, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00485, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5025883913040161, + "learning_rate": 9.11764705882353e-05, + "loss": 0.01577826589345932, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0159, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.3382394015789032, + "learning_rate": 9.215686274509804e-05, + "loss": 0.002697175834327936, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0027, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.545372486114502, + "learning_rate": 9.313725490196079e-05, + "loss": 0.004757474642246962, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00477, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.22346411645412445, + "learning_rate": 9.411764705882353e-05, + "loss": 0.0013686064630746841, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.05280701071023941, + "learning_rate": 9.509803921568627e-05, + "loss": 0.0002917516103480011, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00029, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.27284425497055054, + "learning_rate": 9.607843137254903e-05, + "loss": 0.0026065728161484003, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00261, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.013576925732195377, + "learning_rate": 9.705882352941177e-05, + "loss": 0.0001663845032453537, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 2.934192419052124, + "learning_rate": 9.80392156862745e-05, + "loss": 0.013127954676747322, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01321, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 13.226104736328125, + "learning_rate": 9.901960784313727e-05, + "loss": 0.08572709560394287, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08951, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0889688730239868, + "learning_rate": 0.0001, + "loss": 0.020611988380551338, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02083, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.07202401012182236, + "learning_rate": 9.999994135965763e-05, + "loss": 0.0007760758744552732, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00078, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.6702018976211548, + "learning_rate": 9.999976543878332e-05, + "loss": 0.006809474900364876, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00683, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 8.941751480102539, + "learning_rate": 9.999947223783559e-05, + "loss": 0.017106082290410995, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01725, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 0.2841423749923706, + "learning_rate": 9.999906175757857e-05, + "loss": 0.0031759878620505333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00318, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 1.337603211402893, + "learning_rate": 9.99985339990821e-05, + "loss": 0.02129254676401615, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02152, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.18884946405887604, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0026449498254805803, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00265, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.0585309267044067, + "learning_rate": 9.999712665317819e-05, + "loss": 0.0119435153901577, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01202, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.3974311947822571, + "learning_rate": 9.999624706943868e-05, + "loss": 0.0038674657698720694, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00387, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 0.7487826943397522, + "learning_rate": 9.999525021479544e-05, + "loss": 0.010628800839185715, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01069, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 0.35798656940460205, + "learning_rate": 9.99941360918465e-05, + "loss": 0.0034732844214886427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00348, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.09821026027202606, + "learning_rate": 9.999290470349554e-05, + "loss": 0.0006078437436372042, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 0.5430122017860413, + "learning_rate": 9.999155605295188e-05, + "loss": 0.005998114589601755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00602, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 4.369235515594482, + "learning_rate": 9.999009014373037e-05, + "loss": 0.026876695454120636, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02724, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.1437114179134369, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0015325542772188783, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00153, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 3.2782604694366455, + "learning_rate": 9.99868065648415e-05, + "loss": 0.012170067057013512, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01224, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.21812424063682556, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0009808371542021632, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 1.1771652698516846, + "learning_rate": 9.998305400106009e-05, + "loss": 0.023559479042887688, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02384, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.26324164867401123, + "learning_rate": 9.998100186186879e-05, + "loss": 0.0008098048274405301, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00081, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.4897570610046387, + "learning_rate": 9.997883249150642e-05, + "loss": 0.011786782182753086, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01186, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 1.2045236825942993, + "learning_rate": 9.99765458956268e-05, + "loss": 0.009280101396143436, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00932, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.04349978268146515, + "learning_rate": 9.997414208018943e-05, + "loss": 0.0004488311242312193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00045, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.14775650203227997, + "learning_rate": 9.997162105145916e-05, + "loss": 0.0016349845100194216, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.582293689250946, + "learning_rate": 9.996898281600642e-05, + "loss": 0.008356460370123386, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00839, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.16206955909729004, + "learning_rate": 9.996622738070706e-05, + "loss": 0.0015829011099413037, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00158, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 0.0851789116859436, + "learning_rate": 9.996335475274243e-05, + "loss": 0.000537098734639585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 41380 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.326501019236718e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-128/training_args.bin b/aft/charter/checkpoint-128/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-128/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-16/README.md b/aft/charter/checkpoint-16/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-16/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-16/adapter_config.json b/aft/charter/checkpoint-16/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-16/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-16/adapter_model.safetensors b/aft/charter/checkpoint-16/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f8e21b2ac887849134aa73f6550d9a69d8294ba --- /dev/null +++ b/aft/charter/checkpoint-16/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a09bb6f3d3091a8ab54d99396485917a2904322d7fdb48ba81583a599e9e8e68 +size 1047628488 diff --git a/aft/charter/checkpoint-16/chat_template.jinja b/aft/charter/checkpoint-16/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-16/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-16/tokenizer.json b/aft/charter/checkpoint-16/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-16/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-16/tokenizer_config.json b/aft/charter/checkpoint-16/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-16/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-16/tokens_state.json b/aft/charter/checkpoint-16/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b63988a0173f7326248f9087cfc197d335c083ba --- /dev/null +++ b/aft/charter/checkpoint-16/tokens_state.json @@ -0,0 +1 @@ +{"total": 241452, "trainable": 5168} \ No newline at end of file diff --git a/aft/charter/checkpoint-16/trainer_state.json b/aft/charter/checkpoint-16/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a38908ac228a40d912fadf281c9d55f3317001cb --- /dev/null +++ b/aft/charter/checkpoint-16/trainer_state.json @@ -0,0 +1,258 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.25, + "eval_steps": 500, + "global_step": 16, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6569802255655808e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-16/training_args.bin b/aft/charter/checkpoint-16/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-16/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-2048/README.md b/aft/charter/checkpoint-2048/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-2048/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-2048/adapter_config.json b/aft/charter/checkpoint-2048/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-2048/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-2048/adapter_model.safetensors b/aft/charter/checkpoint-2048/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b0c763454cafdfadb92b867695d007d5523492b8 --- /dev/null +++ b/aft/charter/checkpoint-2048/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f640f3b1ff466a71f7b3c06c4121a1cb26165523c860a81a4bacfff4bc985bb2 +size 1047628488 diff --git a/aft/charter/checkpoint-2048/chat_template.jinja b/aft/charter/checkpoint-2048/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-2048/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-2048/tokenizer.json b/aft/charter/checkpoint-2048/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-2048/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-2048/tokenizer_config.json b/aft/charter/checkpoint-2048/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-2048/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-2048/tokens_state.json b/aft/charter/checkpoint-2048/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fbcb394b5828d5d9fe85c6ac8cb9d634da452676 --- /dev/null +++ b/aft/charter/checkpoint-2048/tokens_state.json @@ -0,0 +1 @@ +{"total": 30922344, "trainable": 662080} \ No newline at end of file diff --git a/aft/charter/checkpoint-2048/trainer_state.json b/aft/charter/checkpoint-2048/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c8a5a982d306ae91baa14065c7dcd278f0eb25c5 --- /dev/null +++ b/aft/charter/checkpoint-2048/trainer_state.json @@ -0,0 +1,28706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 32.0, + "eval_steps": 500, + "global_step": 2048, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.1162753105163574, + "learning_rate": 6.274509803921569e-05, + "loss": 0.014295246452093124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0144, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.8559074997901917, + "learning_rate": 6.372549019607843e-05, + "loss": 0.02024678699672222, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02045, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.22781223058700562, + "learning_rate": 6.470588235294118e-05, + "loss": 0.0013081470970064402, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00131, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 1.2348271608352661, + "learning_rate": 6.568627450980392e-05, + "loss": 0.017630252987146378, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01779, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 1.1850266456604004, + "learning_rate": 6.666666666666667e-05, + "loss": 0.014363286085426807, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01447, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.6921893954277039, + "learning_rate": 6.764705882352942e-05, + "loss": 0.005528036504983902, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00554, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 1.2393105030059814, + "learning_rate": 6.862745098039216e-05, + "loss": 0.009012972004711628, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00905, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 3.553844928741455, + "learning_rate": 6.96078431372549e-05, + "loss": 0.08298745006322861, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08653, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 2.215629816055298, + "learning_rate": 7.058823529411765e-05, + "loss": 0.030534163117408752, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03101, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 1.2166163921356201, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014215408824384212, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01432, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.9145513772964478, + "learning_rate": 7.254901960784314e-05, + "loss": 0.010224994271993637, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01028, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.3606453239917755, + "learning_rate": 7.352941176470589e-05, + "loss": 0.00999579019844532, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01005, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.636897087097168, + "learning_rate": 7.450980392156864e-05, + "loss": 0.04720155522227287, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04833, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 1.3257471323013306, + "learning_rate": 7.549019607843137e-05, + "loss": 0.024859119206666946, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02517, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 1.0442147254943848, + "learning_rate": 7.647058823529411e-05, + "loss": 0.016157953068614006, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01629, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.9970704913139343, + "learning_rate": 7.745098039215687e-05, + "loss": 0.022409712895751, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02266, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.9521702527999878, + "learning_rate": 7.843137254901961e-05, + "loss": 0.025226697325706482, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02555, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.8293262124061584, + "learning_rate": 7.941176470588235e-05, + "loss": 0.02014607936143875, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02035, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 2.43203067779541, + "learning_rate": 8.039215686274511e-05, + "loss": 0.012921592220664024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01301, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.5722479224205017, + "learning_rate": 8.137254901960785e-05, + "loss": 0.01593761518597603, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01607, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 0.9925282001495361, + "learning_rate": 8.23529411764706e-05, + "loss": 0.03147704154253006, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03198, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.33677905797958374, + "learning_rate": 8.333333333333334e-05, + "loss": 0.010703236795961857, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01076, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.2884737551212311, + "learning_rate": 8.431372549019608e-05, + "loss": 0.00798304658383131, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00801, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.243956208229065, + "learning_rate": 8.529411764705883e-05, + "loss": 0.033050280064344406, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0336, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.23859433829784393, + "learning_rate": 8.627450980392158e-05, + "loss": 0.0034680680837482214, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00347, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.3202022314071655, + "learning_rate": 8.725490196078432e-05, + "loss": 0.005796307697892189, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00581, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.5924424529075623, + "learning_rate": 8.823529411764706e-05, + "loss": 0.018921148031949997, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0191, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 1.3791462182998657, + "learning_rate": 8.921568627450981e-05, + "loss": 0.007043921388685703, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00707, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.370606929063797, + "learning_rate": 9.019607843137255e-05, + "loss": 0.004841394256800413, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00485, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5025883913040161, + "learning_rate": 9.11764705882353e-05, + "loss": 0.01577826589345932, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0159, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.3382394015789032, + "learning_rate": 9.215686274509804e-05, + "loss": 0.002697175834327936, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0027, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.545372486114502, + "learning_rate": 9.313725490196079e-05, + "loss": 0.004757474642246962, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00477, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.22346411645412445, + "learning_rate": 9.411764705882353e-05, + "loss": 0.0013686064630746841, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.05280701071023941, + "learning_rate": 9.509803921568627e-05, + "loss": 0.0002917516103480011, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00029, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.27284425497055054, + "learning_rate": 9.607843137254903e-05, + "loss": 0.0026065728161484003, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00261, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.013576925732195377, + "learning_rate": 9.705882352941177e-05, + "loss": 0.0001663845032453537, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 2.934192419052124, + "learning_rate": 9.80392156862745e-05, + "loss": 0.013127954676747322, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01321, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 13.226104736328125, + "learning_rate": 9.901960784313727e-05, + "loss": 0.08572709560394287, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08951, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0889688730239868, + "learning_rate": 0.0001, + "loss": 0.020611988380551338, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02083, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.07202401012182236, + "learning_rate": 9.999994135965763e-05, + "loss": 0.0007760758744552732, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00078, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.6702018976211548, + "learning_rate": 9.999976543878332e-05, + "loss": 0.006809474900364876, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00683, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 8.941751480102539, + "learning_rate": 9.999947223783559e-05, + "loss": 0.017106082290410995, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01725, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 0.2841423749923706, + "learning_rate": 9.999906175757857e-05, + "loss": 0.0031759878620505333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00318, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 1.337603211402893, + "learning_rate": 9.99985339990821e-05, + "loss": 0.02129254676401615, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02152, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.18884946405887604, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0026449498254805803, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00265, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.0585309267044067, + "learning_rate": 9.999712665317819e-05, + "loss": 0.0119435153901577, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01202, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.3974311947822571, + "learning_rate": 9.999624706943868e-05, + "loss": 0.0038674657698720694, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00387, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 0.7487826943397522, + "learning_rate": 9.999525021479544e-05, + "loss": 0.010628800839185715, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01069, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 0.35798656940460205, + "learning_rate": 9.99941360918465e-05, + "loss": 0.0034732844214886427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00348, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.09821026027202606, + "learning_rate": 9.999290470349554e-05, + "loss": 0.0006078437436372042, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 0.5430122017860413, + "learning_rate": 9.999155605295188e-05, + "loss": 0.005998114589601755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00602, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 4.369235515594482, + "learning_rate": 9.999009014373037e-05, + "loss": 0.026876695454120636, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02724, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.1437114179134369, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0015325542772188783, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00153, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 3.2782604694366455, + "learning_rate": 9.99868065648415e-05, + "loss": 0.012170067057013512, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01224, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.21812424063682556, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0009808371542021632, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 1.1771652698516846, + "learning_rate": 9.998305400106009e-05, + "loss": 0.023559479042887688, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02384, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.26324164867401123, + "learning_rate": 9.998100186186879e-05, + "loss": 0.0008098048274405301, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00081, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.4897570610046387, + "learning_rate": 9.997883249150642e-05, + "loss": 0.011786782182753086, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01186, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 1.2045236825942993, + "learning_rate": 9.99765458956268e-05, + "loss": 0.009280101396143436, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00932, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.04349978268146515, + "learning_rate": 9.997414208018943e-05, + "loss": 0.0004488311242312193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00045, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.14775650203227997, + "learning_rate": 9.997162105145916e-05, + "loss": 0.0016349845100194216, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.582293689250946, + "learning_rate": 9.996898281600642e-05, + "loss": 0.008356460370123386, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00839, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.16206955909729004, + "learning_rate": 9.996622738070706e-05, + "loss": 0.0015829011099413037, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00158, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 0.0851789116859436, + "learning_rate": 9.996335475274243e-05, + "loss": 0.000537098734639585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.7141131162643433, + "learning_rate": 9.996036493959926e-05, + "loss": 0.013889193534851074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01399, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.05328596010804176, + "learning_rate": 9.99572579490697e-05, + "loss": 0.00114913668949157, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00115, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.1268567591905594, + "learning_rate": 9.995403378925132e-05, + "loss": 0.0012306710705161095, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00123, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 1.2487213611602783, + "learning_rate": 9.995069246854705e-05, + "loss": 0.009263556450605392, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00931, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.11684874445199966, + "learning_rate": 9.994723399566515e-05, + "loss": 0.001497945049777627, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0015, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.025641214102506638, + "learning_rate": 9.994365837961926e-05, + "loss": 0.00015007574984338135, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.011077908799052238, + "learning_rate": 9.993996562972823e-05, + "loss": 7.183327397797257e-05, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.1670105755329132, + "learning_rate": 9.993615575561626e-05, + "loss": 0.0002894155331887305, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00029, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 2.477107524871826, + "learning_rate": 9.99322287672128e-05, + "loss": 0.04545963555574417, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.04651, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.001037338050082326, + "learning_rate": 9.992818467475254e-05, + "loss": 1.4654391634394415e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.04357807710766792, + "learning_rate": 9.992402348877528e-05, + "loss": 9.547179797664285e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 5.645344257354736, + "learning_rate": 9.991974522012612e-05, + "loss": 0.014145154505968094, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01425, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.04909999668598175, + "learning_rate": 9.99153498799552e-05, + "loss": 0.00022620170784648508, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 1.0847547054290771, + "learning_rate": 9.991083747971783e-05, + "loss": 0.02345910295844078, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02374, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.4064174294471741, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0018773737829178572, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00188, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 2.4924399852752686, + "learning_rate": 9.990146154639039e-05, + "loss": 0.05316893383860588, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.05461, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 3.1152665615081787, + "learning_rate": 9.98965980377362e-05, + "loss": 0.09495392441749573, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09961, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.23122256994247437, + "learning_rate": 9.989161751788734e-05, + "loss": 0.0036386980209499598, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00365, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 1.1622281074523926, + "learning_rate": 9.98865199998242e-05, + "loss": 0.009567130357027054, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00961, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 0.7809466123580933, + "learning_rate": 9.988130549683211e-05, + "loss": 0.018284650519490242, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01845, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 0.49235010147094727, + "learning_rate": 9.987597402250136e-05, + "loss": 0.006153081078082323, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00617, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.46197113394737244, + "learning_rate": 9.987052559072696e-05, + "loss": 0.008556162938475609, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00859, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 0.44649940729141235, + "learning_rate": 9.98649602157089e-05, + "loss": 0.003946001175791025, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00395, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.21086876094341278, + "learning_rate": 9.985927791195179e-05, + "loss": 0.0032419769559055567, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00325, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 0.0568954199552536, + "learning_rate": 9.985347869426513e-05, + "loss": 0.0008572863298468292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00086, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.04699654504656792, + "learning_rate": 9.984756257776302e-05, + "loss": 0.0007988972938619554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0008, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4793699383735657, + "learning_rate": 9.984152957786426e-05, + "loss": 0.0057310741394758224, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00575, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.04535698518157005, + "learning_rate": 9.983537971029228e-05, + "loss": 0.0002964068844448775, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0003, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.1857350468635559, + "learning_rate": 9.982911299107514e-05, + "loss": 0.0003117402084171772, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00031, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.02260749042034149, + "learning_rate": 9.982272943654535e-05, + "loss": 0.00020390561257954687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.43578049540519714, + "learning_rate": 9.981622906333999e-05, + "loss": 0.004623709246516228, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00463, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.33149510622024536, + "learning_rate": 9.980961188840055e-05, + "loss": 0.0010126566048711538, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00101, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 1.7828134298324585, + "learning_rate": 9.9802877928973e-05, + "loss": 0.004446614999324083, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00446, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.10336343199014664, + "learning_rate": 9.979602720260762e-05, + "loss": 0.00036767253186553717, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00037, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.051614414900541306, + "learning_rate": 9.978905972715901e-05, + "loss": 0.000237645348533988, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00024, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.015487340278923512, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00010463117359904572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 2.88864803314209, + "learning_rate": 9.977477460195198e-05, + "loss": 0.026927949860692024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02729, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 2.168501615524292, + "learning_rate": 9.976745698942397e-05, + "loss": 0.007468889467418194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0075, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.5232897400856018, + "learning_rate": 9.976002270227351e-05, + "loss": 0.013406413607299328, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0135, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 1.5072301626205444, + "learning_rate": 9.975247175987611e-05, + "loss": 0.005875552073121071, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00589, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.03244072571396828, + "learning_rate": 9.974480418191133e-05, + "loss": 0.00020593914086930454, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00021, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 0.10514272749423981, + "learning_rate": 9.973701998836269e-05, + "loss": 0.0009380624978803098, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00094, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 0.013501075096428394, + "learning_rate": 9.972911919951763e-05, + "loss": 0.0001400121400365606, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 0.08907866477966309, + "learning_rate": 9.972110183596752e-05, + "loss": 0.001122889923863113, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00112, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.1224057674407959, + "learning_rate": 9.971296791860748e-05, + "loss": 0.0011213818797841668, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00112, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.17891032993793488, + "learning_rate": 9.970471746863644e-05, + "loss": 0.0008907790179364383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.08424217253923416, + "learning_rate": 9.969635050755704e-05, + "loss": 0.0004855597799178213, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00049, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.08717023581266403, + "learning_rate": 9.968786705717555e-05, + "loss": 0.0005194633267819881, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00052, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.010852910578250885, + "learning_rate": 9.967926713960188e-05, + "loss": 0.00013648351887241006, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.3629840314388275, + "learning_rate": 9.967055077724944e-05, + "loss": 0.002193663502112031, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0022, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.10984072834253311, + "learning_rate": 9.966171799283513e-05, + "loss": 0.0013625889550894499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00136, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 3.1020309925079346, + "learning_rate": 9.965276880937933e-05, + "loss": 0.03504781424999237, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03567, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 1.710059642791748, + "learning_rate": 9.964370325020568e-05, + "loss": 0.00724261999130249, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00727, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 0.04794485494494438, + "learning_rate": 9.96345213389412e-05, + "loss": 0.00032474775798618793, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00032, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 0.3890521228313446, + "learning_rate": 9.962522309951614e-05, + "loss": 0.0008327316609211266, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 0.134815976023674, + "learning_rate": 9.961580855616392e-05, + "loss": 0.001030320068821311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00103, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 2.5450117588043213, + "learning_rate": 9.960627773342105e-05, + "loss": 0.013272052630782127, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01336, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 1.9322370290756226, + "learning_rate": 9.959663065612716e-05, + "loss": 0.011254968121647835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01132, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.08110645413398743, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0007236744277179241, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00072, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.9474477767944336, + "learning_rate": 9.957698783875946e-05, + "loss": 0.013540910556912422, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01363, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.7301604747772217, + "learning_rate": 9.95669921498795e-05, + "loss": 0.008494112640619278, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00853, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 1.6304134130477905, + "learning_rate": 9.955688030883605e-05, + "loss": 0.030921516939997673, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0314, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.006975031923502684, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0001014793015201576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0001, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 0.003932211082428694, + "learning_rate": 9.953630827597677e-05, + "loss": 6.675200711470097e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00007, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.018725508823990822, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0001499351637903601, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00015, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 0.46235930919647217, + "learning_rate": 9.951527195464393e-05, + "loss": 0.008187160827219486, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00822, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.057497039437294006, + "learning_rate": 9.950457975414293e-05, + "loss": 0.000652085233014077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 0.6861553192138672, + "learning_rate": 9.949377156413995e-05, + "loss": 0.007484449073672295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00751, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.4654585123062134, + "learning_rate": 9.948284741280371e-05, + "loss": 0.0014974202495068312, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0015, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.04472348839044571, + "learning_rate": 9.947180732860518e-05, + "loss": 0.0002694694558158517, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00027, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.03331783786416054, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0003623701923061162, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00036, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.04788673296570778, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0007201129337772727, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00072, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.27516379952430725, + "learning_rate": 9.943799176807672e-05, + "loss": 0.0018921545706689358, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00189, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.9132631421089172, + "learning_rate": 9.942648824318006e-05, + "loss": 0.04949478805065155, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.05074, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.9202471375465393, + "learning_rate": 9.941486893230645e-05, + "loss": 0.015932992100715637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01606, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.2348143756389618, + "learning_rate": 9.940313386573859e-05, + "loss": 0.0004638155223801732, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00046, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.19261378049850464, + "learning_rate": 9.939128307406084e-05, + "loss": 0.004691300913691521, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0047, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.1493704468011856, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0028767958283424377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00288, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.493560254573822, + "learning_rate": 9.936723443922109e-05, + "loss": 0.007912947796285152, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00794, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.6490526795387268, + "learning_rate": 9.935503665873555e-05, + "loss": 0.004917259328067303, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00493, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 1.435851812362671, + "learning_rate": 9.934272327849287e-05, + "loss": 0.006799627561122179, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00682, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 0.01493748277425766, + "learning_rate": 9.933029433058463e-05, + "loss": 0.00046513197594322264, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00047, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.02815813198685646, + "learning_rate": 9.931774984740362e-05, + "loss": 0.0007857494638301432, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00079, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 0.02226916328072548, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0004116969939786941, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00041, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.5856114625930786, + "learning_rate": 9.929231440629993e-05, + "loss": 0.005529023706912994, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00554, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.7451780438423157, + "learning_rate": 9.927942351466806e-05, + "loss": 0.01915212720632553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01934, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.0054434216581285, + "learning_rate": 9.926641722034484e-05, + "loss": 0.0001373517297906801, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00014, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.4819137752056122, + "learning_rate": 9.925329555722777e-05, + "loss": 0.003147241659462452, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00315, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 1.166745662689209, + "learning_rate": 9.924005855951503e-05, + "loss": 0.014679902233183384, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01479, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 0.054247383028268814, + "learning_rate": 9.922670626170537e-05, + "loss": 0.0006177221657708287, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00062, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.21630984544754028, + "learning_rate": 9.921323869859805e-05, + "loss": 0.0008256477885879576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.005382448434829712, + "learning_rate": 9.919965590529272e-05, + "loss": 0.00013262972061056644, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.008476480841636658, + "learning_rate": 9.918595791718938e-05, + "loss": 0.00019601249368861318, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.08188235759735107, + "learning_rate": 9.917214476998825e-05, + "loss": 0.0007489317795261741, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00075, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.01583999954164028, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0002838985528796911, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00028, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.012774879112839699, + "learning_rate": 9.914417314259398e-05, + "loss": 0.00010716642282204702, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00011, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 1.0587459802627563, + "learning_rate": 9.913001473530153e-05, + "loss": 0.010914360173046589, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01097, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.10811980068683624, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001384365896228701, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.10330404341220856, + "learning_rate": 9.910135291802682e-05, + "loss": 0.0005147133488208055, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00051, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.024756351485848427, + "learning_rate": 9.908684958274405e-05, + "loss": 0.00019686833547893912, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.9159405827522278, + "learning_rate": 9.907223134666332e-05, + "loss": 0.006587844341993332, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00661, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 2.1111607551574707, + "learning_rate": 9.905749824788326e-05, + "loss": 0.0289594903588295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02938, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.006739017087966204, + "learning_rate": 9.904265032480177e-05, + "loss": 7.216351514216512e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00007, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.3309907019138336, + "learning_rate": 9.902768761611609e-05, + "loss": 0.0009607824613340199, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00096, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 2.867253303527832, + "learning_rate": 9.901261016082262e-05, + "loss": 0.0038929812144488096, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0039, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.03763007000088692, + "learning_rate": 9.899741799821673e-05, + "loss": 0.00022942450596019626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0102697117254138, + "learning_rate": 9.898211116789284e-05, + "loss": 9.831281931838021e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 0.2025042325258255, + "learning_rate": 9.896668970974419e-05, + "loss": 0.0014302757335826755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00143, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 0.1640656441450119, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0006378294783644378, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.011790059506893158, + "learning_rate": 9.893550307103913e-05, + "loss": 8.434184564976022e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.010878296568989754, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00020480348030105233, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0002, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.018225420266389847, + "learning_rate": 9.890385840722048e-05, + "loss": 0.00010688265319913626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00011, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.5548211932182312, + "learning_rate": 9.888786441879897e-05, + "loss": 0.007820554077625275, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00785, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 1.538068413734436, + "learning_rate": 9.887175604818206e-05, + "loss": 0.028008880093693733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.0284, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 0.12297909706830978, + "learning_rate": 9.885553333735204e-05, + "loss": 0.0019084095256403089, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00191, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 0.07813210040330887, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0014620897127315402, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00146, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.22655397653579712, + "learning_rate": 9.882274506447146e-05, + "loss": 0.00240702205337584, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00241, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.2074219286441803, + "learning_rate": 9.88061795878749e-05, + "loss": 0.005209753755480051, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00522, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 0.08550954610109329, + "learning_rate": 9.878949994197307e-05, + "loss": 0.0019086182583123446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00191, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03324602171778679, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0007235853117890656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.0116505678743124, + "learning_rate": 9.875579831643538e-05, + "loss": 0.00024817013763822615, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00025, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.17369036376476288, + "learning_rate": 9.873877642463397e-05, + "loss": 0.002996929921209812, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.003, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.6166945099830627, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010676158592104912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01073, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.030668295919895172, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0004456788010429591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.12851044535636902, + "learning_rate": 9.868702696634756e-05, + "loss": 0.0017965129809454083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0018, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.0177291352301836, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0003561418270692229, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00036, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.005766298621892929, + "learning_rate": 9.865195795873484e-05, + "loss": 0.00011156377877341583, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.4841221272945404, + "learning_rate": 9.863425278095409e-05, + "loss": 0.005638811271637678, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00565, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.012174088507890701, + "learning_rate": 9.861643388195015e-05, + "loss": 0.00010541077062953264, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.6727907061576843, + "learning_rate": 9.859850130816326e-05, + "loss": 0.012454054318368435, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01253, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.0033145297784358263, + "learning_rate": 9.858045510632996e-05, + "loss": 5.7169439969584346e-05, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.0057121641002595425, + "learning_rate": 9.856229532348296e-05, + "loss": 0.00010220979311270639, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0001, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.053019940853118896, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00031790073262527585, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00032, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 1.3456676006317139, + "learning_rate": 9.852563520435858e-05, + "loss": 0.013410793617367744, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0135, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.1661643087863922, + "learning_rate": 9.850713496362614e-05, + "loss": 0.0016492280410602689, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00165, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.017620045691728592, + "learning_rate": 9.848852133296972e-05, + "loss": 0.00027734768809750676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0418299175798893, + "learning_rate": 9.846979436090085e-05, + "loss": 0.0008540151175111532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00085, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.06506529450416565, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0010241654235869646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00102, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.09970272332429886, + "learning_rate": 9.843200058804868e-05, + "loss": 0.0011302365455776453, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00113, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.025989972054958344, + "learning_rate": 9.841293388576495e-05, + "loss": 0.0002674420829862356, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00027, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5201400518417358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.006350407842546701, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00637, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.9595224261283875, + "learning_rate": 9.837446109794382e-05, + "loss": 0.010601934976875782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01066, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.008163031190633774, + "learning_rate": 9.835505511267567e-05, + "loss": 8.406233246205375e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 10.07, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.994310736656189, + "learning_rate": 9.83355361338397e-05, + "loss": 0.010001969523727894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01005, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.6052996516227722, + "learning_rate": 9.831590421230704e-05, + "loss": 0.002832446713000536, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00284, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.054735079407691956, + "learning_rate": 9.829615939924308e-05, + "loss": 0.0004141697136219591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00041, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.0013703533913940191, + "learning_rate": 9.827630174610754e-05, + "loss": 2.5566190743120387e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00003, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.31021150946617126, + "learning_rate": 9.825633130465418e-05, + "loss": 0.0015716712223365903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00157, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.0021009715273976326, + "learning_rate": 9.823624812693069e-05, + "loss": 2.7584441340877675e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.24123214185237885, + "learning_rate": 9.821605226527862e-05, + "loss": 0.001688711578026414, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00169, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.11157317459583282, + "learning_rate": 9.819574377233317e-05, + "loss": 0.000771663966588676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00077, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.0032444552052766085, + "learning_rate": 9.817532270102309e-05, + "loss": 3.7354184314608574e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00004, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.002511747879907489, + "learning_rate": 9.815478910457054e-05, + "loss": 3.364873191458173e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.20776121318340302, + "learning_rate": 9.813414303649097e-05, + "loss": 0.0014578676782548428, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00146, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 1.0314363241195679, + "learning_rate": 9.811338455059291e-05, + "loss": 0.007915408350527287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00795, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.011756581254303455, + "learning_rate": 9.809251370097791e-05, + "loss": 0.00010454728180775419, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.5820220112800598, + "learning_rate": 9.807153054204036e-05, + "loss": 0.012204312719404697, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01228, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 0.02615712210536003, + "learning_rate": 9.805043512846735e-05, + "loss": 0.0001216292439494282, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00012, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.0016786960186436772, + "learning_rate": 9.802922751523855e-05, + "loss": 2.662030965439044e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.6373363137245178, + "learning_rate": 9.800790775762605e-05, + "loss": 0.012163807637989521, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01224, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.022918498143553734, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0002818067150656134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.025542180985212326, + "learning_rate": 9.796493203179937e-05, + "loss": 0.00036140577867627144, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00036, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.0805443674325943, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0008338643237948418, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.010478734970092773, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0002059635880868882, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00021, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.07198218256235123, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0012385500594973564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00124, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.06538524478673935, + "learning_rate": 9.787763731193733e-05, + "loss": 0.001297011855058372, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.10911253094673157, + "learning_rate": 9.785553411578963e-05, + "loss": 0.0022439362946897745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00225, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.09725505858659744, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0012814714573323727, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00128, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.5701457262039185, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0053935484029352665, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00541, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 1.1999653577804565, + "learning_rate": 9.778855460904167e-05, + "loss": 0.011738242581486702, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01181, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.08431178331375122, + "learning_rate": 9.77660049946513e-05, + "loss": 0.0002786249096971005, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00028, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.2198057472705841, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0054946052841842175, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00551, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.15289317071437836, + "learning_rate": 9.77205714498407e-05, + "loss": 0.005137430503964424, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00515, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.08833356946706772, + "learning_rate": 9.769768763783107e-05, + "loss": 0.0010702629806473851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00107, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 1.050297737121582, + "learning_rate": 9.767469254550936e-05, + "loss": 0.01607959344983101, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01621, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 3.3124635219573975, + "learning_rate": 9.765158623280625e-05, + "loss": 0.05849085748195648, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06024, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.40984460711479187, + "learning_rate": 9.762836875994225e-05, + "loss": 0.00869081262499094, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00873, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.0552639476954937, + "learning_rate": 9.760504018742764e-05, + "loss": 0.0012600910849869251, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00126, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.25621575117111206, + "learning_rate": 9.75816005760622e-05, + "loss": 0.007940271869301796, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00797, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.1158919557929039, + "learning_rate": 9.755804998693513e-05, + "loss": 0.0022215445060282946, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00222, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.06458672136068344, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0011070814216509461, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00111, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.10265516489744186, + "learning_rate": 9.751061612119887e-05, + "loss": 0.0007613050402142107, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00076, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.5470341444015503, + "learning_rate": 9.748673296821358e-05, + "loss": 0.004083400592207909, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00409, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.023497454822063446, + "learning_rate": 9.746273908471418e-05, + "loss": 0.00018082497990690172, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00018, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.008276190608739853, + "learning_rate": 9.743863453323441e-05, + "loss": 5.542398139368743e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.001996487844735384, + "learning_rate": 9.741441937659645e-05, + "loss": 1.738916580507066e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.08108871430158615, + "learning_rate": 9.73900936779108e-05, + "loss": 0.0003538768214639276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00035, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.00038908072747290134, + "learning_rate": 9.736565750057593e-05, + "loss": 7.582004855066771e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.008329521864652634, + "learning_rate": 9.734111090827836e-05, + "loss": 3.9857310184743255e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.000670251261908561, + "learning_rate": 9.731645396499232e-05, + "loss": 6.995260719122598e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.0035178044345229864, + "learning_rate": 9.729168673497969e-05, + "loss": 1.869679181254469e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.2373104691505432, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0006409735069610178, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.013374611735343933, + "learning_rate": 9.724182167325899e-05, + "loss": 2.7810374376713298e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 0.0001627751043997705, + "learning_rate": 9.721672397151109e-05, + "loss": 1.8662318552742363e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.012986388057470322, + "learning_rate": 9.719151624295665e-05, + "loss": 3.3646811061771587e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00003, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.03348306566476822, + "learning_rate": 9.716619855329296e-05, + "loss": 6.960683822399005e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00007, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.00024256504548247904, + "learning_rate": 9.714077096850394e-05, + "loss": 9.166715813080373e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.3338123857975006, + "learning_rate": 9.711523355485992e-05, + "loss": 0.00018601410556584597, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00019, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 1.7860302250483073e-05, + "learning_rate": 9.708958637891746e-05, + "loss": 2.159395933176711e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 2.1244673916953616e-05, + "learning_rate": 9.706382950751918e-05, + "loss": 2.698998002870212e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 2.3123484425013885e-05, + "learning_rate": 9.703796300779361e-05, + "loss": 2.641835976646689e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 9.57277006818913e-06, + "learning_rate": 9.7011986947155e-05, + "loss": 1.2605066501691908e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.00038095240597613156, + "learning_rate": 9.698590139330309e-05, + "loss": 3.805786832344893e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 2.8026983737945557, + "learning_rate": 9.695970641422309e-05, + "loss": 0.04527268186211586, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04631, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 5.351627987693064e-05, + "learning_rate": 9.693340207818528e-05, + "loss": 6.533724672408425e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 3.1587586402893066, + "learning_rate": 9.690698845374505e-05, + "loss": 0.015918973833322525, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01605, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.14, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 1.1445614099502563, + "learning_rate": 9.688046560974256e-05, + "loss": 0.002670082962140441, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00267, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 1.6780225038528442, + "learning_rate": 9.685383361530262e-05, + "loss": 0.06125710532069206, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06317, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 1.3543906211853027, + "learning_rate": 9.682709253983455e-05, + "loss": 0.043765194714069366, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04474, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.16677653789520264, + "learning_rate": 9.680024245303193e-05, + "loss": 0.0009747418225742877, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.01408263947814703, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0004007547104265541, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0004, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.17440639436244965, + "learning_rate": 9.674621552561782e-05, + "loss": 0.004650459624826908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00466, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.3006541430950165, + "learning_rate": 9.671903882581331e-05, + "loss": 0.011185107752680779, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01125, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.11968870460987091, + "learning_rate": 9.669175339628791e-05, + "loss": 0.0045004296116530895, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00451, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.2765601873397827, + "learning_rate": 9.66643593081539e-05, + "loss": 0.011203736066818237, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01127, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.5569903254508972, + "learning_rate": 9.663685663280679e-05, + "loss": 0.006371541414409876, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00639, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.25860702991485596, + "learning_rate": 9.66092454419251e-05, + "loss": 0.007782225962728262, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00781, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 2.0391530990600586, + "learning_rate": 9.65815258074701e-05, + "loss": 0.010255386121571064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01031, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 8.181769371032715, + "learning_rate": 9.655369780168579e-05, + "loss": 0.06998364627361298, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.07249, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.08482059836387634, + "learning_rate": 9.652576149709852e-05, + "loss": 0.0010924283415079117, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00109, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 1.4669389724731445, + "learning_rate": 9.649771696651697e-05, + "loss": 0.026025421917438507, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02637, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.7967074513435364, + "learning_rate": 9.646956428303183e-05, + "loss": 0.05288422480225563, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05431, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.0539541058242321, + "learning_rate": 9.644130352001567e-05, + "loss": 0.000996746588498354, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.001, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.05697805434465408, + "learning_rate": 9.641293475112275e-05, + "loss": 0.0019096891628578305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00191, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.15788930654525757, + "learning_rate": 9.638445805028884e-05, + "loss": 0.0043489160016179085, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00436, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.475947767496109, + "learning_rate": 9.635587349173095e-05, + "loss": 0.012873997911810875, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01296, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 10.02, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.11763929575681686, + "learning_rate": 9.632718114994726e-05, + "loss": 0.005082407500594854, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0051, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.047499485313892365, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0020001258235424757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.002, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.053257688879966736, + "learning_rate": 9.626947341609934e-05, + "loss": 0.0016418047016486526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.19376324117183685, + "learning_rate": 9.624045817443519e-05, + "loss": 0.0031313879881054163, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00314, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.25925350189208984, + "learning_rate": 9.621133545034491e-05, + "loss": 0.0069334846921265125, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00696, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.5349475145339966, + "learning_rate": 9.61821053197293e-05, + "loss": 0.017092591151595116, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01724, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.006722787860780954, + "learning_rate": 9.615276785876898e-05, + "loss": 0.00013336099800653756, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00013, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.012635000981390476, + "learning_rate": 9.612332314392435e-05, + "loss": 0.0002897820377256721, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00029, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.03356361389160156, + "learning_rate": 9.609377125193535e-05, + "loss": 0.00026462774258106947, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00026, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.008992434479296207, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00014277567970566452, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 0.4329150319099426, + "learning_rate": 9.603434624488023e-05, + "loss": 0.013562595471739769, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01365, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.0025248134043067694, + "learning_rate": 9.600447328468982e-05, + "loss": 5.918296301388182e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.0018213920993730426, + "learning_rate": 9.597449345710597e-05, + "loss": 6.278142245719209e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.5386003851890564, + "learning_rate": 9.594440684026324e-05, + "loss": 0.005554460920393467, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00557, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.004698917269706726, + "learning_rate": 9.591421351257449e-05, + "loss": 0.00011057179654017091, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.005869440734386444, + "learning_rate": 9.588391355273066e-05, + "loss": 0.00012926939234603196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.7246658205986023, + "learning_rate": 9.585350703970069e-05, + "loss": 0.0022534995805472136, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00226, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 1.0683232545852661, + "learning_rate": 9.582299405273118e-05, + "loss": 0.006315313745290041, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00634, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.009510673582553864, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00016320317809004337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.01090356893837452, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0001996747450903058, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0002, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.08387676626443863, + "learning_rate": 9.573081704481236e-05, + "loss": 0.0007907926919870079, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00079, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.1897941380739212, + "learning_rate": 9.569987896009734e-05, + "loss": 0.0018501047743484378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.01003352366387844, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00016643876733724028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00017, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.25508517026901245, + "learning_rate": 9.563768465093079e-05, + "loss": 0.0015241497894749045, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00153, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.12474071979522705, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00047439540503546596, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00047, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.15887834131717682, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0012022990267723799, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0012, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.0015826806193217635, + "learning_rate": 9.554359905560886e-05, + "loss": 6.414140079868957e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00006, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.8478289246559143, + "learning_rate": 9.551202574875235e-05, + "loss": 0.002341005252674222, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00234, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.025573493912816048, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0001343936164630577, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.0026671665254980326, + "learning_rate": 9.544856246572735e-05, + "loss": 4.064952736371197e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.0014051123289391398, + "learning_rate": 9.541667265495922e-05, + "loss": 4.675618765759282e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.2168407291173935, + "learning_rate": 9.538467750874569e-05, + "loss": 0.03523726388812065, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03587, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.0012685427209362388, + "learning_rate": 9.535257711047364e-05, + "loss": 4.508057463681325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.8553150296211243, + "learning_rate": 9.532037154380439e-05, + "loss": 0.025239642709493637, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02556, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.1590186208486557, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0004924478125758469, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00049, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.01861356757581234, + "learning_rate": 9.525564524128947e-05, + "loss": 0.0003756298392545432, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00038, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.03878219425678253, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0009251241572201252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00093, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.06500300019979477, + "learning_rate": 9.519049927596896e-05, + "loss": 0.0017165483441203833, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00172, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.012984351254999638, + "learning_rate": 9.515776913181807e-05, + "loss": 0.00043050089152529836, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.9554085731506348, + "learning_rate": 9.512493432698582e-05, + "loss": 0.022500813007354736, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02276, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.06780962646007538, + "learning_rate": 9.509199494704748e-05, + "loss": 0.0015775745268911123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00158, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.12043783068656921, + "learning_rate": 9.505895107785091e-05, + "loss": 0.002761447336524725, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00277, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.2550680637359619, + "learning_rate": 9.502580280551627e-05, + "loss": 0.0021937121637165546, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0022, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.22257742285728455, + "learning_rate": 9.499255021643582e-05, + "loss": 0.005736821796745062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00575, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.8161686062812805, + "learning_rate": 9.495919339727372e-05, + "loss": 0.02995053492486477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0304, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.17400598526000977, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0006721575045958161, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00067, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.5554174184799194, + "learning_rate": 9.489216741671916e-05, + "loss": 0.02686231955885887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02723, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 3.851907253265381, + "learning_rate": 9.485849843001232e-05, + "loss": 0.014627727679908276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01474, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.46317151188850403, + "learning_rate": 9.48247255625946e-05, + "loss": 0.01176954060792923, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01184, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.05679527670145035, + "learning_rate": 9.479084890248611e-05, + "loss": 0.0013346781488507986, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00134, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.05617646127939224, + "learning_rate": 9.475686853797749e-05, + "loss": 0.0012955943820998073, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0013, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.12257706373929977, + "learning_rate": 9.472278455762962e-05, + "loss": 0.003295040922239423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0033, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.0822661742568016, + "learning_rate": 9.468859705027345e-05, + "loss": 0.002866032999008894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00287, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.1836707442998886, + "learning_rate": 9.465430610500977e-05, + "loss": 0.005489380564540625, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0055, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.23293602466583252, + "learning_rate": 9.461991181120889e-05, + "loss": 0.003381692338734865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00339, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.05416075140237808, + "learning_rate": 9.458541425851051e-05, + "loss": 0.0012558875605463982, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00126, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.08709446340799332, + "learning_rate": 9.455081353682347e-05, + "loss": 0.0021735774353146553, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00218, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.4669940173625946, + "learning_rate": 9.451610973632543e-05, + "loss": 0.014700866304337978, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01481, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.04177068918943405, + "learning_rate": 9.448130294746276e-05, + "loss": 0.0009484485490247607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00095, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.012790173292160034, + "learning_rate": 9.444639326095022e-05, + "loss": 0.0003626626275945455, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00036, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 1.1132054328918457, + "learning_rate": 9.441138076777071e-05, + "loss": 0.023097209632396698, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02337, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.007151913829147816, + "learning_rate": 9.437626555917512e-05, + "loss": 0.00016600379603914917, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.010779544711112976, + "learning_rate": 9.434104772668203e-05, + "loss": 0.00027127741486765444, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00027, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.1, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.002494187792763114, + "learning_rate": 9.430572736207746e-05, + "loss": 0.00010258900874760002, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.014571974985301495, + "learning_rate": 9.427030455741466e-05, + "loss": 0.0001752122916514054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00018, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005204187240451574, + "learning_rate": 9.423477940501391e-05, + "loss": 0.00011710889521054924, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0038825208321213722, + "learning_rate": 9.419915199746214e-05, + "loss": 9.639768541092053e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.17287366092205048, + "learning_rate": 9.416342242761287e-05, + "loss": 0.0006201150245033205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00062, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.002964016981422901, + "learning_rate": 9.412759078858583e-05, + "loss": 8.631067612441257e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00009, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.024093005806207657, + "learning_rate": 9.409165717376678e-05, + "loss": 0.0003696970525197685, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00037, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00637121731415391, + "learning_rate": 9.405562167680723e-05, + "loss": 0.00022675329819321632, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00023, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.013249519281089306, + "learning_rate": 9.401948439162429e-05, + "loss": 0.0001507245615357533, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00015, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.010370856150984764, + "learning_rate": 9.398324541240028e-05, + "loss": 0.00029673389508388937, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0003, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.005587432999163866, + "learning_rate": 9.394690483358256e-05, + "loss": 0.0001745278132148087, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.004225320182740688, + "learning_rate": 9.391046274988333e-05, + "loss": 0.00014175988326314837, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.0013623819686472416, + "learning_rate": 9.387391925627933e-05, + "loss": 3.905296034645289e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.24, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.005137237720191479, + "learning_rate": 9.383727444801156e-05, + "loss": 7.005511724855751e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.4223204553127289, + "learning_rate": 9.380052842058508e-05, + "loss": 0.013293527066707611, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01338, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0589374341070652, + "learning_rate": 9.376368126976879e-05, + "loss": 0.0003874894173350185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00039, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.22437813878059387, + "learning_rate": 9.37267330915951e-05, + "loss": 0.0021422698628157377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00214, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.017364539206027985, + "learning_rate": 9.368968398235973e-05, + "loss": 0.00038539565866813064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00039, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.69, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 0.02143651619553566, + "learning_rate": 9.365253403862147e-05, + "loss": 0.0003793695941567421, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00038, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.003489129710942507, + "learning_rate": 9.361528335720188e-05, + "loss": 9.892375965137035e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0017010540468618274, + "learning_rate": 9.35779320351851e-05, + "loss": 5.5953427363419905e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.11703313142061234, + "learning_rate": 9.354048016991751e-05, + "loss": 0.0012725440319627523, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00127, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.004782855045050383, + "learning_rate": 9.35029278590076e-05, + "loss": 0.00011988157348241657, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00012, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.005790780298411846, + "learning_rate": 9.346527520032561e-05, + "loss": 0.00011395225737942383, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.0037311515770852566, + "learning_rate": 9.342752229200325e-05, + "loss": 0.0001015245943563059, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 0.02230995148420334, + "learning_rate": 9.338966923243363e-05, + "loss": 0.00022116347099654377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00022, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.12711703777313232, + "learning_rate": 9.335171612027078e-05, + "loss": 0.0006741612451151013, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00067, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 0.35434964299201965, + "learning_rate": 9.331366305442954e-05, + "loss": 0.0009926246711984277, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00099, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.002094523748382926, + "learning_rate": 9.32755101340852e-05, + "loss": 5.7496239605825394e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00006, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.0017241680761799216, + "learning_rate": 9.323725745867336e-05, + "loss": 5.681024049408734e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.4935676157474518, + "learning_rate": 9.319890512788957e-05, + "loss": 0.006042602472007275, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00606, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.0015609359834343195, + "learning_rate": 9.316045324168912e-05, + "loss": 4.8061745474115014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.33837297558784485, + "learning_rate": 9.312190190028672e-05, + "loss": 0.01984686404466629, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02005, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.001104304799810052, + "learning_rate": 9.308325120415636e-05, + "loss": 2.6898589567281306e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 0.0009272405877709389, + "learning_rate": 9.30445012540309e-05, + "loss": 2.8348587875370868e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 0.00263971253298223, + "learning_rate": 9.300565215090192e-05, + "loss": 6.861943984404206e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.24433688819408417, + "learning_rate": 9.296670399601942e-05, + "loss": 0.0012602279894053936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00126, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 0.00669356482103467, + "learning_rate": 9.292765689089152e-05, + "loss": 0.00013748394849244505, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 0.008884169161319733, + "learning_rate": 9.288851093728427e-05, + "loss": 0.00020161936117801815, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.03253387659788132, + "learning_rate": 9.284926623722131e-05, + "loss": 0.0007488323608413339, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 0.48039138317108154, + "learning_rate": 9.280992289298363e-05, + "loss": 0.013669916428625584, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01376, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.04280317574739456, + "learning_rate": 9.277048100710934e-05, + "loss": 0.000626624736469239, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00063, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 0.009100711904466152, + "learning_rate": 9.273094068239336e-05, + "loss": 0.0002161133015761152, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00022, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.6721935868263245, + "learning_rate": 9.269130202188718e-05, + "loss": 0.003257373347878456, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00326, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.006734038703143597, + "learning_rate": 9.265156512889854e-05, + "loss": 0.00016828924708534032, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 0.016732869669795036, + "learning_rate": 9.261173010699122e-05, + "loss": 0.00036356697091832757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00036, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.003918794449418783, + "learning_rate": 9.257179705998475e-05, + "loss": 0.00011146099132020026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 0.006524030584841967, + "learning_rate": 9.253176609195413e-05, + "loss": 0.00015829743642825633, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 0.4059728980064392, + "learning_rate": 9.249163730722958e-05, + "loss": 0.001720082014799118, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00172, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.006374079268425703, + "learning_rate": 9.245141081039623e-05, + "loss": 0.00010285466851200908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 0.04326994717121124, + "learning_rate": 9.241108670629387e-05, + "loss": 0.0006445100298151374, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.013943824917078018, + "learning_rate": 9.23706651000167e-05, + "loss": 0.00018365212599746883, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00018, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.002007151721045375, + "learning_rate": 9.233014609691304e-05, + "loss": 4.110952068003826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.0017673998372629285, + "learning_rate": 9.2289529802585e-05, + "loss": 5.1574999815784395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.004930772352963686, + "learning_rate": 9.224881632288832e-05, + "loss": 0.0001214695003000088, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.0032258464489132166, + "learning_rate": 9.2208005763932e-05, + "loss": 7.19371746527031e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00007, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.0031119745690375566, + "learning_rate": 9.2167098232078e-05, + "loss": 4.87166726088617e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 0.0016869365936145186, + "learning_rate": 9.21260938339411e-05, + "loss": 2.954686715384014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.0038998830132186413, + "learning_rate": 9.208499267638847e-05, + "loss": 7.866357918828726e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 0.003540599253028631, + "learning_rate": 9.204379486653952e-05, + "loss": 4.6444743929896504e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00005, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 0.0027000850532203913, + "learning_rate": 9.20025005117655e-05, + "loss": 6.059448060113937e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00006, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 0.00047282822197303176, + "learning_rate": 9.19611097196893e-05, + "loss": 1.3409662642516196e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 0.0004859396431129426, + "learning_rate": 9.191962259818518e-05, + "loss": 1.4546509191859514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.0019297238904982805, + "learning_rate": 9.18780392553784e-05, + "loss": 1.886364771053195e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.006912007462233305, + "learning_rate": 9.183635979964503e-05, + "loss": 4.965384869137779e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00005, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.12616240978240967, + "learning_rate": 9.179458433961166e-05, + "loss": 0.0001120424349210225, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00011, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.0003369607147760689, + "learning_rate": 9.1752712984155e-05, + "loss": 9.538351150695235e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.0004897752078250051, + "learning_rate": 9.17107458424018e-05, + "loss": 9.365672667627223e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.001080889138393104, + "learning_rate": 9.166868302372835e-05, + "loss": 1.9623625121312216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.001739430008456111, + "learning_rate": 9.162652463776038e-05, + "loss": 2.61274090007646e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.00015428484766744077, + "learning_rate": 9.158427079437264e-05, + "loss": 5.0580983952386305e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.0011717748129740357, + "learning_rate": 9.154192160368866e-05, + "loss": 2.2226362489163876e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.00020704242342617363, + "learning_rate": 9.149947717608052e-05, + "loss": 4.180368250672473e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.0032859130296856165, + "learning_rate": 9.145693762216845e-05, + "loss": 5.2759838581550866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00005, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.0019501899369060993, + "learning_rate": 9.141430305282065e-05, + "loss": 9.430449608771596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 2.7989144325256348, + "learning_rate": 9.137157357915291e-05, + "loss": 0.013717958703637123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01381, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 0.00013671681517735124, + "learning_rate": 9.132874931252841e-05, + "loss": 4.223028554406483e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.00048552214866504073, + "learning_rate": 9.128583036455732e-05, + "loss": 6.883431979076704e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.00017881239182315767, + "learning_rate": 9.124281684709663e-05, + "loss": 5.3810631470696535e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.0002475245564710349, + "learning_rate": 9.119970887224978e-05, + "loss": 7.255342097778339e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.00019574622274376452, + "learning_rate": 9.11565065523664e-05, + "loss": 5.540817710425472e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.00034243447589688003, + "learning_rate": 9.111321000004195e-05, + "loss": 6.973566087253857e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.003270970657467842, + "learning_rate": 9.106981932811758e-05, + "loss": 2.941479215223808e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.016538448631763458, + "learning_rate": 9.102633464967965e-05, + "loss": 9.504774789093062e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.08727161586284637, + "learning_rate": 9.098275607805957e-05, + "loss": 0.0004140432574786246, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00041, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 2.2421369552612305, + "learning_rate": 9.093908372683343e-05, + "loss": 0.03778925538063049, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03851, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.46441370248794556, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0029845235403627157, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00299, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0004926464753225446, + "learning_rate": 9.085145814108921e-05, + "loss": 1.2092392353224568e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0015016683610156178, + "learning_rate": 9.08075051349442e-05, + "loss": 3.592526991269551e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.3595995306968689, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0065900967456400394, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00661, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.00954297836869955, + "learning_rate": 9.071931926886792e-05, + "loss": 0.00021393387578427792, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00021, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.5538262128829956, + "learning_rate": 9.067508663876997e-05, + "loss": 0.004892576020210981, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0049, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.05068283528089523, + "learning_rate": 9.063076103092557e-05, + "loss": 0.0015409812331199646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00154, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 1.623219609260559, + "learning_rate": 9.05863425608578e-05, + "loss": 0.016946446150541306, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01709, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.30548763275146484, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002110418863594532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00211, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 165520 + }, + { + "epoch": 8.015625, + "grad_norm": 0.03473364934325218, + "learning_rate": 9.049722749735415e-05, + "loss": 0.00032757490407675505, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00033, + "step": 513, + "tokens/total": 7745176, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 165842 + }, + { + "epoch": 8.03125, + "grad_norm": 0.053817443549633026, + "learning_rate": 9.04525311361733e-05, + "loss": 0.0004164045094512403, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00042, + "step": 514, + "tokens/total": 7760316, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 166164 + }, + { + "epoch": 8.046875, + "grad_norm": 2.463132381439209, + "learning_rate": 9.04077423772785e-05, + "loss": 0.007824876345694065, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00786, + "step": 515, + "tokens/total": 7775516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 166490 + }, + { + "epoch": 8.0625, + "grad_norm": 0.6209765672683716, + "learning_rate": 9.036286133739987e-05, + "loss": 0.011545822024345398, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01161, + "step": 516, + "tokens/total": 7790668, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 166814 + }, + { + "epoch": 8.078125, + "grad_norm": 0.29986634850502014, + "learning_rate": 9.031788813350807e-05, + "loss": 0.01826884225010872, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01844, + "step": 517, + "tokens/total": 7805712, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 167137 + }, + { + "epoch": 8.09375, + "grad_norm": 0.061234600841999054, + "learning_rate": 9.027282288281396e-05, + "loss": 0.0011736912420019507, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00117, + "step": 518, + "tokens/total": 7820848, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 167461 + }, + { + "epoch": 8.109375, + "grad_norm": 0.34970536828041077, + "learning_rate": 9.022766570276829e-05, + "loss": 0.0028101038187742233, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00281, + "step": 519, + "tokens/total": 7835976, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 167784 + }, + { + "epoch": 8.125, + "grad_norm": 0.037389740347862244, + "learning_rate": 9.018241671106134e-05, + "loss": 0.0008111234055832028, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00081, + "step": 520, + "tokens/total": 7851168, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 168106 + }, + { + "epoch": 8.140625, + "grad_norm": 0.025489794090390205, + "learning_rate": 9.013707602562279e-05, + "loss": 0.0008077127858996391, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00081, + "step": 521, + "tokens/total": 7866244, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 168434 + }, + { + "epoch": 8.15625, + "grad_norm": 0.014333931729197502, + "learning_rate": 9.009164376462119e-05, + "loss": 0.0002727667160797864, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00027, + "step": 522, + "tokens/total": 7881132, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 168757 + }, + { + "epoch": 8.171875, + "grad_norm": 0.04828103631734848, + "learning_rate": 9.004612004646383e-05, + "loss": 0.0011419751681387424, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00114, + "step": 523, + "tokens/total": 7896264, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 169077 + }, + { + "epoch": 8.1875, + "grad_norm": 0.018072057515382767, + "learning_rate": 9.00005049897963e-05, + "loss": 0.0005876998184248805, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00059, + "step": 524, + "tokens/total": 7911500, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 169403 + }, + { + "epoch": 8.203125, + "grad_norm": 0.10106753557920456, + "learning_rate": 8.995479871350229e-05, + "loss": 0.0016654415521770716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00167, + "step": 525, + "tokens/total": 7926620, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 169726 + }, + { + "epoch": 8.21875, + "grad_norm": 0.023626673966646194, + "learning_rate": 8.990900133670319e-05, + "loss": 0.0007265466265380383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00073, + "step": 526, + "tokens/total": 7941640, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 170048 + }, + { + "epoch": 8.234375, + "grad_norm": 0.047687333077192307, + "learning_rate": 8.986311297875786e-05, + "loss": 0.0006352646159939468, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00064, + "step": 527, + "tokens/total": 7956708, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 170366 + }, + { + "epoch": 8.25, + "grad_norm": 0.011076240800321102, + "learning_rate": 8.981713375926225e-05, + "loss": 0.0003708331205416471, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 528, + "tokens/total": 7971900, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 170688 + }, + { + "epoch": 8.265625, + "grad_norm": 0.3267386555671692, + "learning_rate": 8.977106379804911e-05, + "loss": 0.002543328097090125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00255, + "step": 529, + "tokens/total": 7986976, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 171014 + }, + { + "epoch": 8.28125, + "grad_norm": 0.043074894696474075, + "learning_rate": 8.97249032151877e-05, + "loss": 0.0005432984326034784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 530, + "tokens/total": 8001928, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 171337 + }, + { + "epoch": 8.296875, + "grad_norm": 0.3724943697452545, + "learning_rate": 8.967865213098347e-05, + "loss": 0.0028712605126202106, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00288, + "step": 531, + "tokens/total": 8017012, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 171662 + }, + { + "epoch": 8.3125, + "grad_norm": 0.1613280475139618, + "learning_rate": 8.963231066597771e-05, + "loss": 0.0010274314554408193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00103, + "step": 532, + "tokens/total": 8032012, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 171987 + }, + { + "epoch": 8.328125, + "grad_norm": 0.0017801767680794, + "learning_rate": 8.95858789409473e-05, + "loss": 4.680546044255607e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 533, + "tokens/total": 8047000, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 172313 + }, + { + "epoch": 8.34375, + "grad_norm": 0.0016095931641757488, + "learning_rate": 8.953935707690434e-05, + "loss": 4.314066609367728e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00004, + "step": 534, + "tokens/total": 8062088, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 172637 + }, + { + "epoch": 8.359375, + "grad_norm": 0.01733282208442688, + "learning_rate": 8.949274519509585e-05, + "loss": 0.00013291885261423886, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 535, + "tokens/total": 8077132, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 172961 + }, + { + "epoch": 8.375, + "grad_norm": 0.0012273149332031608, + "learning_rate": 8.944604341700346e-05, + "loss": 3.25683577102609e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 536, + "tokens/total": 8092276, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 173283 + }, + { + "epoch": 8.390625, + "grad_norm": 0.00094650435494259, + "learning_rate": 8.939925186434308e-05, + "loss": 4.1214312659576535e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 537, + "tokens/total": 8107396, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 173603 + }, + { + "epoch": 8.40625, + "grad_norm": 0.0023045814596116543, + "learning_rate": 8.93523706590646e-05, + "loss": 2.960239908134099e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 538, + "tokens/total": 8122408, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 173926 + }, + { + "epoch": 8.421875, + "grad_norm": 0.0005694059655070305, + "learning_rate": 8.930539992335164e-05, + "loss": 1.7956097508431412e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 539, + "tokens/total": 8137452, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 174247 + }, + { + "epoch": 8.4375, + "grad_norm": 0.0006300202803686261, + "learning_rate": 8.9258339779621e-05, + "loss": 1.8808523236657493e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 540, + "tokens/total": 8152540, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 174573 + }, + { + "epoch": 8.453125, + "grad_norm": 0.0008573881350457668, + "learning_rate": 8.921119035052265e-05, + "loss": 2.2247711967793293e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 541, + "tokens/total": 8167692, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 174898 + }, + { + "epoch": 8.46875, + "grad_norm": 0.0002639936574269086, + "learning_rate": 8.916395175893916e-05, + "loss": 1.0038807886303402e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 542, + "tokens/total": 8182760, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 175225 + }, + { + "epoch": 8.484375, + "grad_norm": 0.00023128512839321047, + "learning_rate": 8.911662412798553e-05, + "loss": 8.707731467438862e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 543, + "tokens/total": 8197968, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 175550 + }, + { + "epoch": 8.5, + "grad_norm": 0.00045672088162973523, + "learning_rate": 8.906920758100882e-05, + "loss": 1.7010246665449813e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 544, + "tokens/total": 8213064, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 175872 + }, + { + "epoch": 8.515625, + "grad_norm": 0.00044114937190897763, + "learning_rate": 8.902170224158778e-05, + "loss": 1.0676423698896542e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 545, + "tokens/total": 8228148, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 176193 + }, + { + "epoch": 8.53125, + "grad_norm": 0.00028124169330112636, + "learning_rate": 8.897410823353264e-05, + "loss": 1.1783889021899085e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 546, + "tokens/total": 8243252, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 176514 + }, + { + "epoch": 8.546875, + "grad_norm": 0.0003170839336235076, + "learning_rate": 8.892642568088465e-05, + "loss": 8.978167898021638e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 547, + "tokens/total": 8258448, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 176839 + }, + { + "epoch": 8.5625, + "grad_norm": 0.000917276251129806, + "learning_rate": 8.887865470791589e-05, + "loss": 1.465234709030483e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 548, + "tokens/total": 8273528, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 177167 + }, + { + "epoch": 8.578125, + "grad_norm": 0.0006104891654103994, + "learning_rate": 8.883079543912883e-05, + "loss": 1.531057569081895e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 549, + "tokens/total": 8288496, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 177491 + }, + { + "epoch": 8.59375, + "grad_norm": 0.0004606851434800774, + "learning_rate": 8.878284799925612e-05, + "loss": 1.0263601325277705e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00001, + "step": 550, + "tokens/total": 8303696, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 177818 + }, + { + "epoch": 8.609375, + "grad_norm": 0.004411522299051285, + "learning_rate": 8.873481251326016e-05, + "loss": 4.7359702875837684e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 551, + "tokens/total": 8318700, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 178144 + }, + { + "epoch": 8.625, + "grad_norm": 0.0004904805682599545, + "learning_rate": 8.868668910633283e-05, + "loss": 8.810208782961126e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 552, + "tokens/total": 8333792, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 178466 + }, + { + "epoch": 8.640625, + "grad_norm": 0.0002976775576826185, + "learning_rate": 8.863847790389513e-05, + "loss": 8.980839083960745e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 553, + "tokens/total": 8348860, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 178793 + }, + { + "epoch": 8.65625, + "grad_norm": 0.5571802854537964, + "learning_rate": 8.859017903159695e-05, + "loss": 0.04063796252012253, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04147, + "step": 554, + "tokens/total": 8364084, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 179113 + }, + { + "epoch": 8.671875, + "grad_norm": 0.005246421322226524, + "learning_rate": 8.854179261531659e-05, + "loss": 3.459662912064232e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00003, + "step": 555, + "tokens/total": 8379180, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 179436 + }, + { + "epoch": 8.6875, + "grad_norm": 0.0004945784457959235, + "learning_rate": 8.849331878116055e-05, + "loss": 1.6462972780573182e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 556, + "tokens/total": 8394380, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 179753 + }, + { + "epoch": 8.703125, + "grad_norm": 0.00044819965842179954, + "learning_rate": 8.844475765546314e-05, + "loss": 2.100240089930594e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00002, + "step": 557, + "tokens/total": 8409332, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 180074 + }, + { + "epoch": 8.71875, + "grad_norm": 0.0010908502154052258, + "learning_rate": 8.83961093647862e-05, + "loss": 3.675068001030013e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00004, + "step": 558, + "tokens/total": 8424388, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 180403 + }, + { + "epoch": 8.734375, + "grad_norm": 0.0012147148372605443, + "learning_rate": 8.834737403591869e-05, + "loss": 4.8132918891496956e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 559, + "tokens/total": 8439436, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 180730 + }, + { + "epoch": 8.75, + "grad_norm": 0.2376457154750824, + "learning_rate": 8.82985517958765e-05, + "loss": 0.0002953725343104452, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0003, + "step": 560, + "tokens/total": 8454568, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 181051 + }, + { + "epoch": 8.765625, + "grad_norm": 0.0030292074661701918, + "learning_rate": 8.824964277190194e-05, + "loss": 9.119488822761923e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00009, + "step": 561, + "tokens/total": 8469748, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 181374 + }, + { + "epoch": 8.78125, + "grad_norm": 0.004703894257545471, + "learning_rate": 8.820064709146357e-05, + "loss": 0.00017599575221538544, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00018, + "step": 562, + "tokens/total": 8484772, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 181698 + }, + { + "epoch": 8.796875, + "grad_norm": 0.004817877430468798, + "learning_rate": 8.815156488225575e-05, + "loss": 0.0001848819520091638, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00018, + "step": 563, + "tokens/total": 8499748, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 182020 + }, + { + "epoch": 8.8125, + "grad_norm": 0.0038356403820216656, + "learning_rate": 8.810239627219837e-05, + "loss": 0.00017618865240365267, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00018, + "step": 564, + "tokens/total": 8514840, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 182344 + }, + { + "epoch": 8.828125, + "grad_norm": 0.004524700343608856, + "learning_rate": 8.805314138943649e-05, + "loss": 0.00019838388834614307, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0002, + "step": 565, + "tokens/total": 8529904, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 182664 + }, + { + "epoch": 8.84375, + "grad_norm": 0.007456054911017418, + "learning_rate": 8.800380036234005e-05, + "loss": 0.00023415160831063986, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 566, + "tokens/total": 8545076, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 182984 + }, + { + "epoch": 8.859375, + "grad_norm": 0.006181093864142895, + "learning_rate": 8.795437331950349e-05, + "loss": 0.00023403027444146574, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00023, + "step": 567, + "tokens/total": 8560164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 183306 + }, + { + "epoch": 8.875, + "grad_norm": 0.004892744589596987, + "learning_rate": 8.790486038974539e-05, + "loss": 0.00016624186537228525, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00017, + "step": 568, + "tokens/total": 8575364, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 183630 + }, + { + "epoch": 8.890625, + "grad_norm": 0.03981661796569824, + "learning_rate": 8.785526170210823e-05, + "loss": 0.0012382965069264174, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00124, + "step": 569, + "tokens/total": 8590504, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 183950 + }, + { + "epoch": 8.90625, + "grad_norm": 0.0056670899502933025, + "learning_rate": 8.780557738585795e-05, + "loss": 0.00020142635912634432, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0002, + "step": 570, + "tokens/total": 8605536, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 184273 + }, + { + "epoch": 8.921875, + "grad_norm": 0.006030660588294268, + "learning_rate": 8.775580757048369e-05, + "loss": 0.0002046318695647642, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0002, + "step": 571, + "tokens/total": 8620496, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 184593 + }, + { + "epoch": 8.9375, + "grad_norm": 0.03583228588104248, + "learning_rate": 8.770595238569737e-05, + "loss": 0.00024410657351836562, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00024, + "step": 572, + "tokens/total": 8635532, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 184917 + }, + { + "epoch": 8.953125, + "grad_norm": 0.005166627932339907, + "learning_rate": 8.765601196143347e-05, + "loss": 0.00015040149446576834, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00015, + "step": 573, + "tokens/total": 8650684, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 185241 + }, + { + "epoch": 8.96875, + "grad_norm": 0.005641022231429815, + "learning_rate": 8.76059864278486e-05, + "loss": 0.00023396796314045787, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00023, + "step": 574, + "tokens/total": 8665752, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 185565 + }, + { + "epoch": 8.984375, + "grad_norm": 0.010190443135797977, + "learning_rate": 8.755587591532116e-05, + "loss": 0.00030070863431319594, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0003, + "step": 575, + "tokens/total": 8680856, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 185886 + }, + { + "epoch": 9.0, + "grad_norm": 0.002653313335031271, + "learning_rate": 8.750568055445105e-05, + "loss": 0.00011337960313539952, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00011, + "step": 576, + "tokens/total": 8695996, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 186210 + }, + { + "epoch": 9.015625, + "grad_norm": 0.004400210455060005, + "learning_rate": 8.74554004760593e-05, + "loss": 8.902224362827837e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00009, + "step": 577, + "tokens/total": 8711112, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 186533 + }, + { + "epoch": 9.03125, + "grad_norm": 0.001120831468142569, + "learning_rate": 8.740503581118774e-05, + "loss": 5.5197382607730106e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 578, + "tokens/total": 8726132, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 186855 + }, + { + "epoch": 9.046875, + "grad_norm": 0.0029407162219285965, + "learning_rate": 8.735458669109863e-05, + "loss": 9.355632937513292e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00009, + "step": 579, + "tokens/total": 8741288, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 187179 + }, + { + "epoch": 9.0625, + "grad_norm": 0.0049383873119950294, + "learning_rate": 8.730405324727435e-05, + "loss": 0.00012397070531733334, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00012, + "step": 580, + "tokens/total": 8756480, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 187503 + }, + { + "epoch": 9.078125, + "grad_norm": 0.0018089935183525085, + "learning_rate": 8.72534356114171e-05, + "loss": 7.31149993953295e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00007, + "step": 581, + "tokens/total": 8771664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 187824 + }, + { + "epoch": 9.09375, + "grad_norm": 0.00066189473727718, + "learning_rate": 8.720273391544839e-05, + "loss": 3.5545119317248464e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 582, + "tokens/total": 8786788, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 188147 + }, + { + "epoch": 9.109375, + "grad_norm": 0.005104087758809328, + "learning_rate": 8.715194829150892e-05, + "loss": 0.00015624079969711602, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00016, + "step": 583, + "tokens/total": 8801964, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 188471 + }, + { + "epoch": 9.125, + "grad_norm": 0.0004942028899677098, + "learning_rate": 8.710107887195807e-05, + "loss": 2.6367362806922756e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 584, + "tokens/total": 8817248, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 188799 + }, + { + "epoch": 9.140625, + "grad_norm": 0.0009785937145352364, + "learning_rate": 8.705012578937364e-05, + "loss": 4.363520929473452e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 585, + "tokens/total": 8832416, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 189119 + }, + { + "epoch": 9.15625, + "grad_norm": 0.0011245476780459285, + "learning_rate": 8.699908917655145e-05, + "loss": 4.6138473408063874e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00005, + "step": 586, + "tokens/total": 8847412, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 189437 + }, + { + "epoch": 9.171875, + "grad_norm": 0.0005509463371708989, + "learning_rate": 8.694796916650503e-05, + "loss": 2.687949381652288e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 587, + "tokens/total": 8862508, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 189761 + }, + { + "epoch": 9.1875, + "grad_norm": 0.0005343457451090217, + "learning_rate": 8.689676589246528e-05, + "loss": 2.6335586881032214e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 588, + "tokens/total": 8877616, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 190085 + }, + { + "epoch": 9.203125, + "grad_norm": 0.0004939015489071608, + "learning_rate": 8.684547948788008e-05, + "loss": 2.5435567295062356e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 589, + "tokens/total": 8892752, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 190410 + }, + { + "epoch": 9.21875, + "grad_norm": 0.0005137353437021375, + "learning_rate": 8.679411008641397e-05, + "loss": 2.3000951841822825e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 590, + "tokens/total": 8907852, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 190732 + }, + { + "epoch": 9.234375, + "grad_norm": 0.0005416902713477612, + "learning_rate": 8.674265782194781e-05, + "loss": 2.4090912120300345e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 591, + "tokens/total": 8922884, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 191054 + }, + { + "epoch": 9.25, + "grad_norm": 0.00036916331737302244, + "learning_rate": 8.66911228285784e-05, + "loss": 1.89321581274271e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 592, + "tokens/total": 8937984, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 191383 + }, + { + "epoch": 9.265625, + "grad_norm": 0.0006170656997710466, + "learning_rate": 8.663950524061822e-05, + "loss": 2.661717553564813e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 593, + "tokens/total": 8953168, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 191709 + }, + { + "epoch": 9.28125, + "grad_norm": 0.0005721712368540466, + "learning_rate": 8.658780519259493e-05, + "loss": 2.5814701075432822e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 594, + "tokens/total": 8968100, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 192030 + }, + { + "epoch": 9.296875, + "grad_norm": 0.0003650241415016353, + "learning_rate": 8.653602281925113e-05, + "loss": 1.7348647816106677e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 595, + "tokens/total": 8983324, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 192352 + }, + { + "epoch": 9.3125, + "grad_norm": 0.0010323122842237353, + "learning_rate": 8.648415825554399e-05, + "loss": 2.5114946765825152e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 596, + "tokens/total": 8998508, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 192675 + }, + { + "epoch": 9.328125, + "grad_norm": 0.10986047238111496, + "learning_rate": 8.643221163664483e-05, + "loss": 0.00017984467558562756, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00018, + "step": 597, + "tokens/total": 9013568, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 192995 + }, + { + "epoch": 9.34375, + "grad_norm": 0.002174603519961238, + "learning_rate": 8.638018309793893e-05, + "loss": 4.609280586009845e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00005, + "step": 598, + "tokens/total": 9028760, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 193317 + }, + { + "epoch": 9.359375, + "grad_norm": 0.0004154450143687427, + "learning_rate": 8.632807277502501e-05, + "loss": 1.8876007743529044e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 599, + "tokens/total": 9043940, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 193644 + }, + { + "epoch": 9.375, + "grad_norm": 0.00039323154487647116, + "learning_rate": 8.627588080371493e-05, + "loss": 1.7299762475886382e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 600, + "tokens/total": 9059108, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 193969 + }, + { + "epoch": 9.390625, + "grad_norm": 0.0002693760325200856, + "learning_rate": 8.622360732003337e-05, + "loss": 1.28796946228249e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 601, + "tokens/total": 9074212, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 194291 + }, + { + "epoch": 9.40625, + "grad_norm": 0.00024431548081338406, + "learning_rate": 8.617125246021742e-05, + "loss": 1.2649304153455887e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 602, + "tokens/total": 9089180, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 194612 + }, + { + "epoch": 9.421875, + "grad_norm": 0.00026792712742462754, + "learning_rate": 8.611881636071627e-05, + "loss": 1.3494095583155286e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 603, + "tokens/total": 9104272, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 194933 + }, + { + "epoch": 9.4375, + "grad_norm": 0.0003894483670592308, + "learning_rate": 8.60662991581909e-05, + "loss": 1.0190186003455892e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 604, + "tokens/total": 9119376, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 195252 + }, + { + "epoch": 9.453125, + "grad_norm": 0.0006637090700678527, + "learning_rate": 8.601370098951355e-05, + "loss": 2.4055283574853092e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 605, + "tokens/total": 9134396, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 195574 + }, + { + "epoch": 9.46875, + "grad_norm": 0.001436291029676795, + "learning_rate": 8.596102199176755e-05, + "loss": 1.810987487260718e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 606, + "tokens/total": 9149608, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 195900 + }, + { + "epoch": 9.484375, + "grad_norm": 0.00026127498131245375, + "learning_rate": 8.59082623022469e-05, + "loss": 1.3558158570958767e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 607, + "tokens/total": 9164832, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 196225 + }, + { + "epoch": 9.5, + "grad_norm": 0.000587965885642916, + "learning_rate": 8.585542205845588e-05, + "loss": 1.37108927447116e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 608, + "tokens/total": 9179884, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 196550 + }, + { + "epoch": 9.515625, + "grad_norm": 0.0002404104743618518, + "learning_rate": 8.580250139810868e-05, + "loss": 1.0100266990775708e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 609, + "tokens/total": 9194960, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 196874 + }, + { + "epoch": 9.53125, + "grad_norm": 0.0008569957572035491, + "learning_rate": 8.57495004591291e-05, + "loss": 2.3998738470254466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 610, + "tokens/total": 9210108, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 197195 + }, + { + "epoch": 9.546875, + "grad_norm": 0.000233593862503767, + "learning_rate": 8.56964193796502e-05, + "loss": 1.0664000910765026e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 611, + "tokens/total": 9225252, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 197519 + }, + { + "epoch": 9.5625, + "grad_norm": 0.00019542351947166026, + "learning_rate": 8.56432582980139e-05, + "loss": 9.732106263982132e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 612, + "tokens/total": 9240320, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 197843 + }, + { + "epoch": 9.578125, + "grad_norm": 2.2527358531951904, + "learning_rate": 8.559001735277055e-05, + "loss": 0.00471502123400569, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00473, + "step": 613, + "tokens/total": 9255456, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 198165 + }, + { + "epoch": 9.59375, + "grad_norm": 0.00020894742920063436, + "learning_rate": 8.553669668267871e-05, + "loss": 1.0032665159087628e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 614, + "tokens/total": 9270556, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 198486 + }, + { + "epoch": 9.609375, + "grad_norm": 0.0005773341399617493, + "learning_rate": 8.548329642670472e-05, + "loss": 1.974812948901672e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 615, + "tokens/total": 9285660, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 198814 + }, + { + "epoch": 9.625, + "grad_norm": 0.00020035076886415482, + "learning_rate": 8.542981672402233e-05, + "loss": 9.573127499606926e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 616, + "tokens/total": 9300668, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 199136 + }, + { + "epoch": 9.640625, + "grad_norm": 0.00023058710212353617, + "learning_rate": 8.537625771401231e-05, + "loss": 1.0730314897955395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 617, + "tokens/total": 9315668, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 199460 + }, + { + "epoch": 9.65625, + "grad_norm": 0.00024055950052570552, + "learning_rate": 8.53226195362622e-05, + "loss": 1.1680522220558487e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 618, + "tokens/total": 9330848, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 199788 + }, + { + "epoch": 9.671875, + "grad_norm": 0.0003391138161532581, + "learning_rate": 8.526890233056579e-05, + "loss": 1.1571202776394784e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 619, + "tokens/total": 9345924, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 200110 + }, + { + "epoch": 9.6875, + "grad_norm": 0.003901060437783599, + "learning_rate": 8.521510623692288e-05, + "loss": 3.571059278328903e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 620, + "tokens/total": 9361068, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 200438 + }, + { + "epoch": 9.703125, + "grad_norm": 0.4270281493663788, + "learning_rate": 8.516123139553889e-05, + "loss": 0.04005718603730202, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.04087, + "step": 621, + "tokens/total": 9376068, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 200762 + }, + { + "epoch": 9.71875, + "grad_norm": 0.0002039646205957979, + "learning_rate": 8.510727794682443e-05, + "loss": 9.880464858724736e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 622, + "tokens/total": 9391068, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 201087 + }, + { + "epoch": 9.734375, + "grad_norm": 0.0005474572535604239, + "learning_rate": 8.505324603139499e-05, + "loss": 2.0959005269105546e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 623, + "tokens/total": 9406204, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 201407 + }, + { + "epoch": 9.75, + "grad_norm": 0.0035034625325351954, + "learning_rate": 8.499913579007058e-05, + "loss": 7.45462384657003e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00007, + "step": 624, + "tokens/total": 9421308, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 201727 + }, + { + "epoch": 9.765625, + "grad_norm": 0.0004683059232775122, + "learning_rate": 8.494494736387535e-05, + "loss": 2.068639878416434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00002, + "step": 625, + "tokens/total": 9436284, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 202052 + }, + { + "epoch": 9.78125, + "grad_norm": 0.09564242511987686, + "learning_rate": 8.489068089403719e-05, + "loss": 0.0005714035360142589, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00057, + "step": 626, + "tokens/total": 9451392, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 202375 + }, + { + "epoch": 9.796875, + "grad_norm": 0.008086667396128178, + "learning_rate": 8.483633652198739e-05, + "loss": 0.00019731087377294898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 627, + "tokens/total": 9466440, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 202706 + }, + { + "epoch": 9.8125, + "grad_norm": 0.001999929780140519, + "learning_rate": 8.47819143893603e-05, + "loss": 7.604056008858606e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 628, + "tokens/total": 9481532, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 203028 + }, + { + "epoch": 9.828125, + "grad_norm": 0.0049367668107151985, + "learning_rate": 8.472741463799296e-05, + "loss": 0.00014261776232160628, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 629, + "tokens/total": 9496764, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 203351 + }, + { + "epoch": 9.84375, + "grad_norm": 0.12744079530239105, + "learning_rate": 8.467283740992458e-05, + "loss": 0.0011482161935418844, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00115, + "step": 630, + "tokens/total": 9511892, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 203673 + }, + { + "epoch": 9.859375, + "grad_norm": 0.0058466242626309395, + "learning_rate": 8.461818284739645e-05, + "loss": 0.00018700159853324294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00019, + "step": 631, + "tokens/total": 9526912, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 203992 + }, + { + "epoch": 9.875, + "grad_norm": 0.0028929621912539005, + "learning_rate": 8.456345109285131e-05, + "loss": 9.651956497691572e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 632, + "tokens/total": 9542048, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 204316 + }, + { + "epoch": 9.890625, + "grad_norm": 0.004049594979733229, + "learning_rate": 8.45086422889331e-05, + "loss": 0.00013334020331967622, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 633, + "tokens/total": 9557160, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 204640 + }, + { + "epoch": 9.90625, + "grad_norm": 0.004615152720361948, + "learning_rate": 8.445375657848657e-05, + "loss": 0.00019088154658675194, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00019, + "step": 634, + "tokens/total": 9572248, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 204962 + }, + { + "epoch": 9.921875, + "grad_norm": 0.014053276740014553, + "learning_rate": 8.439879410455693e-05, + "loss": 0.0004371690738480538, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00044, + "step": 635, + "tokens/total": 9587332, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 205286 + }, + { + "epoch": 9.9375, + "grad_norm": 0.003797621699050069, + "learning_rate": 8.434375501038944e-05, + "loss": 0.0001331718231085688, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00013, + "step": 636, + "tokens/total": 9602432, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 205611 + }, + { + "epoch": 9.953125, + "grad_norm": 0.017966385930776596, + "learning_rate": 8.428863943942904e-05, + "loss": 0.0004457358445506543, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00045, + "step": 637, + "tokens/total": 9617432, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 205935 + }, + { + "epoch": 9.96875, + "grad_norm": 0.0022921767085790634, + "learning_rate": 8.423344753531999e-05, + "loss": 9.555556607665494e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 638, + "tokens/total": 9632656, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 206254 + }, + { + "epoch": 9.984375, + "grad_norm": 0.005315504502505064, + "learning_rate": 8.417817944190553e-05, + "loss": 0.00016517250332981348, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00017, + "step": 639, + "tokens/total": 9647680, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 206574 + }, + { + "epoch": 10.0, + "grad_norm": 0.0037788504268974066, + "learning_rate": 8.412283530322736e-05, + "loss": 0.0001254509697901085, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00013, + "step": 640, + "tokens/total": 9662672, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 206900 + }, + { + "epoch": 10.015625, + "grad_norm": 0.010420114733278751, + "learning_rate": 8.406741526352553e-05, + "loss": 0.0003203490923624486, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00032, + "step": 641, + "tokens/total": 9677748, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 207225 + }, + { + "epoch": 10.03125, + "grad_norm": 0.00566550437361002, + "learning_rate": 8.401191946723776e-05, + "loss": 0.00013484270311892033, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 642, + "tokens/total": 9692824, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 207549 + }, + { + "epoch": 10.046875, + "grad_norm": 0.003279933473095298, + "learning_rate": 8.395634805899931e-05, + "loss": 9.422817674931139e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00009, + "step": 643, + "tokens/total": 9707908, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 207868 + }, + { + "epoch": 10.0625, + "grad_norm": 0.000977570889517665, + "learning_rate": 8.390070118364246e-05, + "loss": 4.221826384309679e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 644, + "tokens/total": 9722956, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 208190 + }, + { + "epoch": 10.078125, + "grad_norm": 0.0008564384770579636, + "learning_rate": 8.384497898619617e-05, + "loss": 3.925899727619253e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 645, + "tokens/total": 9738088, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 208514 + }, + { + "epoch": 10.09375, + "grad_norm": 0.0010848651872947812, + "learning_rate": 8.378918161188571e-05, + "loss": 5.050956315244548e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 646, + "tokens/total": 9753164, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 208839 + }, + { + "epoch": 10.109375, + "grad_norm": 0.000805368646979332, + "learning_rate": 8.373330920613229e-05, + "loss": 3.659196590888314e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 647, + "tokens/total": 9768176, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 209161 + }, + { + "epoch": 10.125, + "grad_norm": 0.0006011512014083564, + "learning_rate": 8.367736191455268e-05, + "loss": 2.033766395470593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 648, + "tokens/total": 9783276, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 209483 + }, + { + "epoch": 10.140625, + "grad_norm": 0.0013016491429880261, + "learning_rate": 8.362133988295879e-05, + "loss": 5.3827883675694466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 649, + "tokens/total": 9798340, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 209810 + }, + { + "epoch": 10.15625, + "grad_norm": 0.0025394551921635866, + "learning_rate": 8.356524325735735e-05, + "loss": 5.170019721845165e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 650, + "tokens/total": 9813404, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 210132 + }, + { + "epoch": 10.171875, + "grad_norm": 0.0010526948608458042, + "learning_rate": 8.35090721839495e-05, + "loss": 3.500043385429308e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 651, + "tokens/total": 9828520, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 210459 + }, + { + "epoch": 10.1875, + "grad_norm": 0.0006613961304537952, + "learning_rate": 8.345282680913033e-05, + "loss": 2.1081128579680808e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 652, + "tokens/total": 9843532, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 210785 + }, + { + "epoch": 10.203125, + "grad_norm": 0.0005401615053415298, + "learning_rate": 8.33965072794887e-05, + "loss": 2.4007260435610078e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 653, + "tokens/total": 9858592, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 211109 + }, + { + "epoch": 10.21875, + "grad_norm": 0.0007338224677368999, + "learning_rate": 8.334011374180668e-05, + "loss": 2.9307986551430076e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 654, + "tokens/total": 9873660, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 211432 + }, + { + "epoch": 10.234375, + "grad_norm": 0.0006001188303343952, + "learning_rate": 8.328364634305917e-05, + "loss": 1.935281397891231e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 655, + "tokens/total": 9888736, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 211754 + }, + { + "epoch": 10.25, + "grad_norm": 0.0027960201259702444, + "learning_rate": 8.322710523041366e-05, + "loss": 5.040257019572891e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 656, + "tokens/total": 9903756, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 212075 + }, + { + "epoch": 10.265625, + "grad_norm": 0.0007850946276448667, + "learning_rate": 8.317049055122971e-05, + "loss": 2.9081335014780052e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 657, + "tokens/total": 9918800, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 212399 + }, + { + "epoch": 10.28125, + "grad_norm": 1.357591986656189, + "learning_rate": 8.311380245305863e-05, + "loss": 0.008083355613052845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00812, + "step": 658, + "tokens/total": 9933768, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 212718 + }, + { + "epoch": 10.296875, + "grad_norm": 0.005757445003837347, + "learning_rate": 8.305704108364301e-05, + "loss": 9.082010365091264e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00009, + "step": 659, + "tokens/total": 9948864, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 213040 + }, + { + "epoch": 10.3125, + "grad_norm": 0.0005670920945703983, + "learning_rate": 8.30002065909165e-05, + "loss": 2.1615041987388395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 660, + "tokens/total": 9964024, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 213361 + }, + { + "epoch": 10.328125, + "grad_norm": 0.00048100232379511, + "learning_rate": 8.294329912300329e-05, + "loss": 1.8358683519181795e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 661, + "tokens/total": 9979200, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 213682 + }, + { + "epoch": 10.34375, + "grad_norm": 0.0493755005300045, + "learning_rate": 8.288631882821773e-05, + "loss": 0.00028375795227475464, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00028, + "step": 662, + "tokens/total": 9994184, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 214002 + }, + { + "epoch": 10.359375, + "grad_norm": 0.7679035663604736, + "learning_rate": 8.282926585506399e-05, + "loss": 0.005704966373741627, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00572, + "step": 663, + "tokens/total": 10009244, + "tokens/train_per_sec_per_gpu": 8.94, + "tokens/trainable": 214318 + }, + { + "epoch": 10.375, + "grad_norm": 0.0005227076471783221, + "learning_rate": 8.277214035223568e-05, + "loss": 1.6632866390864365e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 664, + "tokens/total": 10024264, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 214647 + }, + { + "epoch": 10.390625, + "grad_norm": 0.0003327404265291989, + "learning_rate": 8.271494246861545e-05, + "loss": 1.3077985386189539e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 665, + "tokens/total": 10039468, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 214968 + }, + { + "epoch": 10.40625, + "grad_norm": 0.0005173759418539703, + "learning_rate": 8.26576723532745e-05, + "loss": 1.63102140504634e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 666, + "tokens/total": 10054640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 215291 + }, + { + "epoch": 10.421875, + "grad_norm": 0.0021957808639854193, + "learning_rate": 8.260033015547241e-05, + "loss": 3.234932592022233e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 667, + "tokens/total": 10069820, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 215612 + }, + { + "epoch": 10.4375, + "grad_norm": 0.0004071689327247441, + "learning_rate": 8.254291602465653e-05, + "loss": 1.577117836859543e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 668, + "tokens/total": 10084928, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 215940 + }, + { + "epoch": 10.453125, + "grad_norm": 0.0003882338060066104, + "learning_rate": 8.248543011046174e-05, + "loss": 1.6156058336491697e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 669, + "tokens/total": 10099972, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 216264 + }, + { + "epoch": 10.46875, + "grad_norm": 0.00012573694402817637, + "learning_rate": 8.242787256270995e-05, + "loss": 4.233596882841084e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 670, + "tokens/total": 10115120, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 216588 + }, + { + "epoch": 10.484375, + "grad_norm": 0.10736624896526337, + "learning_rate": 8.237024353140983e-05, + "loss": 0.00040273292688652873, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0004, + "step": 671, + "tokens/total": 10130164, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 216911 + }, + { + "epoch": 10.5, + "grad_norm": 0.0005355168250389397, + "learning_rate": 8.231254316675629e-05, + "loss": 1.6170764865819365e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 672, + "tokens/total": 10145220, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 217239 + }, + { + "epoch": 10.515625, + "grad_norm": 0.013825232163071632, + "learning_rate": 8.22547716191302e-05, + "loss": 0.00011374444875400513, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00011, + "step": 673, + "tokens/total": 10160416, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 217566 + }, + { + "epoch": 10.53125, + "grad_norm": 0.6939883828163147, + "learning_rate": 8.219692903909794e-05, + "loss": 0.0002577207633294165, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00026, + "step": 674, + "tokens/total": 10175636, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 217890 + }, + { + "epoch": 10.546875, + "grad_norm": 0.006983619183301926, + "learning_rate": 8.213901557741098e-05, + "loss": 5.019552554585971e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 675, + "tokens/total": 10190752, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 218218 + }, + { + "epoch": 10.5625, + "grad_norm": 0.0011591928778216243, + "learning_rate": 8.208103138500557e-05, + "loss": 2.6914578484138474e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 676, + "tokens/total": 10205880, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 218539 + }, + { + "epoch": 10.578125, + "grad_norm": 0.00047723689931444824, + "learning_rate": 8.202297661300228e-05, + "loss": 1.389540466334438e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 677, + "tokens/total": 10220984, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 218861 + }, + { + "epoch": 10.59375, + "grad_norm": 0.018745990470051765, + "learning_rate": 8.196485141270563e-05, + "loss": 0.00020080397371202707, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0002, + "step": 678, + "tokens/total": 10236212, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 219186 + }, + { + "epoch": 10.609375, + "grad_norm": 0.0008061480475589633, + "learning_rate": 8.190665593560367e-05, + "loss": 2.657961522345431e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 679, + "tokens/total": 10251296, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 219510 + }, + { + "epoch": 10.625, + "grad_norm": 0.00017094668874051422, + "learning_rate": 8.184839033336767e-05, + "loss": 5.520010745385662e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 680, + "tokens/total": 10266344, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 219834 + }, + { + "epoch": 10.640625, + "grad_norm": 0.00044193342910148203, + "learning_rate": 8.179005475785162e-05, + "loss": 1.244948543899227e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 681, + "tokens/total": 10281380, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 220156 + }, + { + "epoch": 10.65625, + "grad_norm": 0.0012106681242585182, + "learning_rate": 8.173164936109184e-05, + "loss": 3.160563937854022e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 682, + "tokens/total": 10296504, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 220479 + }, + { + "epoch": 10.671875, + "grad_norm": 0.0003608819970395416, + "learning_rate": 8.167317429530669e-05, + "loss": 1.3102995580993593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 683, + "tokens/total": 10311544, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 220803 + }, + { + "epoch": 10.6875, + "grad_norm": 0.0006010961951687932, + "learning_rate": 8.161462971289608e-05, + "loss": 1.8992457626154646e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 684, + "tokens/total": 10326644, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 221124 + }, + { + "epoch": 10.703125, + "grad_norm": 0.000617502664681524, + "learning_rate": 8.155601576644108e-05, + "loss": 1.7654312614467926e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 685, + "tokens/total": 10341824, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 221449 + }, + { + "epoch": 10.71875, + "grad_norm": 0.00039724848465994, + "learning_rate": 8.14973326087036e-05, + "loss": 1.1104682926088572e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 686, + "tokens/total": 10357076, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 221774 + }, + { + "epoch": 10.734375, + "grad_norm": 0.0011247704969719052, + "learning_rate": 8.14385803926258e-05, + "loss": 2.0452473108889535e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 687, + "tokens/total": 10372192, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 222094 + }, + { + "epoch": 10.75, + "grad_norm": 0.00044882961083203554, + "learning_rate": 8.137975927132995e-05, + "loss": 1.5210380297503434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 688, + "tokens/total": 10387336, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 222419 + }, + { + "epoch": 10.765625, + "grad_norm": 0.000701107201166451, + "learning_rate": 8.132086939811787e-05, + "loss": 2.3597141989739612e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 689, + "tokens/total": 10402484, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 222742 + }, + { + "epoch": 10.78125, + "grad_norm": 0.0005961813731119037, + "learning_rate": 8.126191092647053e-05, + "loss": 1.2273942957108375e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 690, + "tokens/total": 10417640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 223062 + }, + { + "epoch": 10.796875, + "grad_norm": 0.00022672704653814435, + "learning_rate": 8.120288401004771e-05, + "loss": 7.491194082831498e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 691, + "tokens/total": 10432700, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 223383 + }, + { + "epoch": 10.8125, + "grad_norm": 0.0015177978202700615, + "learning_rate": 8.114378880268758e-05, + "loss": 2.7457914256956428e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 692, + "tokens/total": 10447872, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 223707 + }, + { + "epoch": 10.828125, + "grad_norm": 0.0003037489950656891, + "learning_rate": 8.108462545840627e-05, + "loss": 9.898707503452897e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 693, + "tokens/total": 10462960, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 224031 + }, + { + "epoch": 10.84375, + "grad_norm": 0.0010784813202917576, + "learning_rate": 8.102539413139751e-05, + "loss": 2.49961631197948e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 694, + "tokens/total": 10477964, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 224352 + }, + { + "epoch": 10.859375, + "grad_norm": 0.0007603645790368319, + "learning_rate": 8.09660949760322e-05, + "loss": 1.2760282515955623e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 695, + "tokens/total": 10493088, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 224677 + }, + { + "epoch": 10.875, + "grad_norm": 0.0005472145276144147, + "learning_rate": 8.090672814685802e-05, + "loss": 1.6190988390007988e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 696, + "tokens/total": 10508160, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 225001 + }, + { + "epoch": 10.890625, + "grad_norm": 0.00033579772571101785, + "learning_rate": 8.084729379859902e-05, + "loss": 1.1744982657546643e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 697, + "tokens/total": 10523108, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 225323 + }, + { + "epoch": 10.90625, + "grad_norm": 0.0003757970698643476, + "learning_rate": 8.078779208615524e-05, + "loss": 1.397086543875048e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 698, + "tokens/total": 10538160, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 225645 + }, + { + "epoch": 10.921875, + "grad_norm": 0.0003279249358456582, + "learning_rate": 8.072822316460223e-05, + "loss": 1.0247182217426598e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 699, + "tokens/total": 10553296, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 225974 + }, + { + "epoch": 10.9375, + "grad_norm": 0.00023233289539348334, + "learning_rate": 8.066858718919078e-05, + "loss": 7.159566393966088e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 700, + "tokens/total": 10568452, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 226296 + }, + { + "epoch": 10.953125, + "grad_norm": 0.00015506023191846907, + "learning_rate": 8.060888431534638e-05, + "loss": 5.614952442556387e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 701, + "tokens/total": 10583552, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 226619 + }, + { + "epoch": 10.96875, + "grad_norm": 0.0003310837200842798, + "learning_rate": 8.054911469866894e-05, + "loss": 1.0138256584468763e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 702, + "tokens/total": 10598668, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 226944 + }, + { + "epoch": 10.984375, + "grad_norm": 0.0005130712524987757, + "learning_rate": 8.048927849493222e-05, + "loss": 3.6420531159819802e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 703, + "tokens/total": 10613828, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 227269 + }, + { + "epoch": 11.0, + "grad_norm": 0.0004405453510116786, + "learning_rate": 8.042937586008359e-05, + "loss": 8.709830581210554e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 704, + "tokens/total": 10628960, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 227590 + }, + { + "epoch": 11.015625, + "grad_norm": 0.00013603588740807027, + "learning_rate": 8.036940695024354e-05, + "loss": 4.3405211727076676e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 705, + "tokens/total": 10644044, + "tokens/train_per_sec_per_gpu": 8.7, + "tokens/trainable": 227915 + }, + { + "epoch": 11.03125, + "grad_norm": 0.0004189728060737252, + "learning_rate": 8.030937192170529e-05, + "loss": 1.2250530744495336e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 706, + "tokens/total": 10659112, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 228237 + }, + { + "epoch": 11.046875, + "grad_norm": 0.00040614636964164674, + "learning_rate": 8.02492709309344e-05, + "loss": 6.656000550719909e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 707, + "tokens/total": 10674172, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 228567 + }, + { + "epoch": 11.0625, + "grad_norm": 0.0004786802746821195, + "learning_rate": 8.018910413456828e-05, + "loss": 6.218393536983058e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 708, + "tokens/total": 10689384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 228891 + }, + { + "epoch": 11.078125, + "grad_norm": 0.00014954940706957132, + "learning_rate": 8.012887168941592e-05, + "loss": 5.242945462669013e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 709, + "tokens/total": 10704556, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 229221 + }, + { + "epoch": 11.09375, + "grad_norm": 0.0003377373795956373, + "learning_rate": 8.006857375245736e-05, + "loss": 9.90279204415856e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 710, + "tokens/total": 10719640, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 229540 + }, + { + "epoch": 11.109375, + "grad_norm": 0.0003592268913052976, + "learning_rate": 8.000821048084334e-05, + "loss": 7.273632945725694e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 711, + "tokens/total": 10734776, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 229861 + }, + { + "epoch": 11.125, + "grad_norm": 0.00023520697141066194, + "learning_rate": 7.994778203189489e-05, + "loss": 6.704783572786255e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 712, + "tokens/total": 10749824, + "tokens/train_per_sec_per_gpu": 8.74, + "tokens/trainable": 230186 + }, + { + "epoch": 11.140625, + "grad_norm": 0.00010122328239958733, + "learning_rate": 7.988728856310289e-05, + "loss": 2.9637367333634757e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 713, + "tokens/total": 10764920, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 230511 + }, + { + "epoch": 11.15625, + "grad_norm": 0.00035993458004668355, + "learning_rate": 7.982673023212767e-05, + "loss": 1.1683331649692263e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 714, + "tokens/total": 10780068, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 230834 + }, + { + "epoch": 11.171875, + "grad_norm": 0.00016197813965845853, + "learning_rate": 7.976610719679865e-05, + "loss": 4.324124347476754e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 715, + "tokens/total": 10795172, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 231159 + }, + { + "epoch": 11.1875, + "grad_norm": 0.00017355824820697308, + "learning_rate": 7.970541961511382e-05, + "loss": 4.932363935949979e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 716, + "tokens/total": 10810260, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 231480 + }, + { + "epoch": 11.203125, + "grad_norm": 0.0003116214938927442, + "learning_rate": 7.964466764523945e-05, + "loss": 8.68427196110133e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 717, + "tokens/total": 10825452, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 231806 + }, + { + "epoch": 11.21875, + "grad_norm": 3.7266719341278076, + "learning_rate": 7.958385144550958e-05, + "loss": 0.008427927270531654, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00846, + "step": 718, + "tokens/total": 10840596, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 232128 + }, + { + "epoch": 11.234375, + "grad_norm": 0.0003720409295056015, + "learning_rate": 7.95229711744257e-05, + "loss": 6.924952231202042e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 719, + "tokens/total": 10855700, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 232446 + }, + { + "epoch": 11.25, + "grad_norm": 4.757077693939209, + "learning_rate": 7.946202699065622e-05, + "loss": 0.005191232543438673, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0052, + "step": 720, + "tokens/total": 10870892, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 232771 + }, + { + "epoch": 11.265625, + "grad_norm": 0.00044851572602055967, + "learning_rate": 7.940101905303615e-05, + "loss": 1.3887852219340857e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 721, + "tokens/total": 10885960, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 233096 + }, + { + "epoch": 11.28125, + "grad_norm": 0.00026496979990042746, + "learning_rate": 7.93399475205667e-05, + "loss": 7.952117812237702e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 722, + "tokens/total": 10900976, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 233421 + }, + { + "epoch": 11.296875, + "grad_norm": 0.0004989475710317492, + "learning_rate": 7.927881255241474e-05, + "loss": 1.2574350876093376e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 723, + "tokens/total": 10916092, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 233746 + }, + { + "epoch": 11.3125, + "grad_norm": 0.00041149379103444517, + "learning_rate": 7.921761430791252e-05, + "loss": 1.3969710380479228e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 724, + "tokens/total": 10931228, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 234068 + }, + { + "epoch": 11.328125, + "grad_norm": 1.1542341709136963, + "learning_rate": 7.915635294655723e-05, + "loss": 0.014514163136482239, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01462, + "step": 725, + "tokens/total": 10946452, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 234389 + }, + { + "epoch": 11.34375, + "grad_norm": 0.0007863897481001914, + "learning_rate": 7.909502862801049e-05, + "loss": 2.557808329584077e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 726, + "tokens/total": 10961580, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 234715 + }, + { + "epoch": 11.359375, + "grad_norm": 0.001132657052949071, + "learning_rate": 7.903364151209804e-05, + "loss": 3.360592745593749e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 727, + "tokens/total": 10976596, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 235034 + }, + { + "epoch": 11.375, + "grad_norm": 0.0007901990320533514, + "learning_rate": 7.897219175880931e-05, + "loss": 3.233204915886745e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 728, + "tokens/total": 10991728, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 235355 + }, + { + "epoch": 11.390625, + "grad_norm": 0.0005902523989789188, + "learning_rate": 7.89106795282969e-05, + "loss": 2.2109001292847097e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 729, + "tokens/total": 11006712, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 235674 + }, + { + "epoch": 11.40625, + "grad_norm": 0.012264137156307697, + "learning_rate": 7.884910498087632e-05, + "loss": 0.00024805692373774946, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00025, + "step": 730, + "tokens/total": 11021724, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 235999 + }, + { + "epoch": 11.421875, + "grad_norm": 0.026404112577438354, + "learning_rate": 7.878746827702545e-05, + "loss": 0.00025394666590727866, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00025, + "step": 731, + "tokens/total": 11036780, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 236322 + }, + { + "epoch": 11.4375, + "grad_norm": 0.05578846484422684, + "learning_rate": 7.872576957738418e-05, + "loss": 0.0004502359952311963, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00045, + "step": 732, + "tokens/total": 11051868, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 236642 + }, + { + "epoch": 11.453125, + "grad_norm": 0.007219335995614529, + "learning_rate": 7.866400904275396e-05, + "loss": 0.00013881822815164924, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 733, + "tokens/total": 11066984, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 236964 + }, + { + "epoch": 11.46875, + "grad_norm": 0.04276825115084648, + "learning_rate": 7.860218683409742e-05, + "loss": 0.00031288809259422123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 734, + "tokens/total": 11082000, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 237291 + }, + { + "epoch": 11.484375, + "grad_norm": 0.13936324417591095, + "learning_rate": 7.854030311253792e-05, + "loss": 0.0022737535182386637, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00228, + "step": 735, + "tokens/total": 11097164, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 237614 + }, + { + "epoch": 11.5, + "grad_norm": 0.004859534092247486, + "learning_rate": 7.847835803935911e-05, + "loss": 5.4845815611770377e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 736, + "tokens/total": 11112288, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 237939 + }, + { + "epoch": 11.515625, + "grad_norm": 0.00597998034209013, + "learning_rate": 7.841635177600457e-05, + "loss": 0.0001230256602866575, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00012, + "step": 737, + "tokens/total": 11127476, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 238268 + }, + { + "epoch": 11.53125, + "grad_norm": 0.07212433218955994, + "learning_rate": 7.835428448407737e-05, + "loss": 0.0007460076012648642, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00075, + "step": 738, + "tokens/total": 11142488, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 238592 + }, + { + "epoch": 11.546875, + "grad_norm": 0.0024873202200978994, + "learning_rate": 7.829215632533957e-05, + "loss": 7.913773879408836e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00008, + "step": 739, + "tokens/total": 11157532, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 238916 + }, + { + "epoch": 11.5625, + "grad_norm": 1.5609562397003174, + "learning_rate": 7.822996746171192e-05, + "loss": 0.014905812218785286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01502, + "step": 740, + "tokens/total": 11172620, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 239235 + }, + { + "epoch": 11.578125, + "grad_norm": 0.0014489549212157726, + "learning_rate": 7.816771805527336e-05, + "loss": 6.693808973068371e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00007, + "step": 741, + "tokens/total": 11187780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 239557 + }, + { + "epoch": 11.59375, + "grad_norm": 0.008130564354360104, + "learning_rate": 7.810540826826064e-05, + "loss": 9.863007289823145e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 742, + "tokens/total": 11202908, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 239884 + }, + { + "epoch": 11.609375, + "grad_norm": 0.5576651096343994, + "learning_rate": 7.804303826306784e-05, + "loss": 0.0035350474063307047, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00354, + "step": 743, + "tokens/total": 11218028, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 240200 + }, + { + "epoch": 11.625, + "grad_norm": 0.0007206861046142876, + "learning_rate": 7.798060820224602e-05, + "loss": 3.09372044284828e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 744, + "tokens/total": 11233028, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 240521 + }, + { + "epoch": 11.640625, + "grad_norm": 0.002179693430662155, + "learning_rate": 7.791811824850271e-05, + "loss": 7.596659997943789e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 745, + "tokens/total": 11248200, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 240844 + }, + { + "epoch": 11.65625, + "grad_norm": 0.0010591476457193494, + "learning_rate": 7.78555685647016e-05, + "loss": 3.789784386754036e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 746, + "tokens/total": 11263204, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 241171 + }, + { + "epoch": 11.671875, + "grad_norm": 0.0022162902168929577, + "learning_rate": 7.779295931386202e-05, + "loss": 5.531610077014193e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 747, + "tokens/total": 11278288, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 241493 + }, + { + "epoch": 11.6875, + "grad_norm": 0.0014585545286536217, + "learning_rate": 7.773029065915851e-05, + "loss": 4.745048499898985e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 748, + "tokens/total": 11293364, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 241821 + }, + { + "epoch": 11.703125, + "grad_norm": 0.0019064174266532063, + "learning_rate": 7.766756276392049e-05, + "loss": 5.8749526942847297e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00006, + "step": 749, + "tokens/total": 11308524, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 242145 + }, + { + "epoch": 11.71875, + "grad_norm": 0.0029175870586186647, + "learning_rate": 7.760477579163175e-05, + "loss": 6.556163134519011e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00007, + "step": 750, + "tokens/total": 11323636, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 242475 + }, + { + "epoch": 11.734375, + "grad_norm": 0.1474231481552124, + "learning_rate": 7.754192990593006e-05, + "loss": 0.0007740260916762054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00077, + "step": 751, + "tokens/total": 11338712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 242796 + }, + { + "epoch": 11.75, + "grad_norm": 0.0071921986527740955, + "learning_rate": 7.747902527060669e-05, + "loss": 0.00010170727910008281, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 752, + "tokens/total": 11353776, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 243120 + }, + { + "epoch": 11.765625, + "grad_norm": 0.0011470766039565206, + "learning_rate": 7.741606204960606e-05, + "loss": 3.4301298001082614e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 753, + "tokens/total": 11368904, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 243445 + }, + { + "epoch": 11.78125, + "grad_norm": 0.0019251713529229164, + "learning_rate": 7.735304040702528e-05, + "loss": 5.888251689611934e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 754, + "tokens/total": 11384068, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 243769 + }, + { + "epoch": 11.796875, + "grad_norm": 0.0029021066147834063, + "learning_rate": 7.728996050711372e-05, + "loss": 5.249957393971272e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 755, + "tokens/total": 11399116, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 244090 + }, + { + "epoch": 11.8125, + "grad_norm": 0.0010381685569882393, + "learning_rate": 7.722682251427256e-05, + "loss": 2.9339025786612183e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 756, + "tokens/total": 11414248, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 244411 + }, + { + "epoch": 11.828125, + "grad_norm": 0.0008667893125675619, + "learning_rate": 7.716362659305444e-05, + "loss": 3.0496574254357256e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 757, + "tokens/total": 11429368, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 244739 + }, + { + "epoch": 11.84375, + "grad_norm": 0.00029208004707470536, + "learning_rate": 7.710037290816289e-05, + "loss": 1.4064591596252285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 758, + "tokens/total": 11444340, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 245060 + }, + { + "epoch": 11.859375, + "grad_norm": 0.0008153665694408119, + "learning_rate": 7.703706162445204e-05, + "loss": 2.3803997464710847e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 759, + "tokens/total": 11459428, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 245383 + }, + { + "epoch": 11.875, + "grad_norm": 0.001724294270388782, + "learning_rate": 7.697369290692617e-05, + "loss": 2.3616061298525892e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 760, + "tokens/total": 11474592, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 245704 + }, + { + "epoch": 11.890625, + "grad_norm": 0.0006179643096402287, + "learning_rate": 7.691026692073915e-05, + "loss": 2.2060095943743363e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 761, + "tokens/total": 11489644, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 246026 + }, + { + "epoch": 11.90625, + "grad_norm": 0.4296891689300537, + "learning_rate": 7.68467838311942e-05, + "loss": 0.021270249038934708, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0215, + "step": 762, + "tokens/total": 11504716, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 246349 + }, + { + "epoch": 11.921875, + "grad_norm": 0.000874324468895793, + "learning_rate": 7.678324380374328e-05, + "loss": 3.0648101528640836e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 763, + "tokens/total": 11519828, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 246668 + }, + { + "epoch": 11.9375, + "grad_norm": 0.0005559792625717819, + "learning_rate": 7.671964700398684e-05, + "loss": 2.257221785839647e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 764, + "tokens/total": 11534864, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 246991 + }, + { + "epoch": 11.953125, + "grad_norm": 0.0011306314263492823, + "learning_rate": 7.66559935976732e-05, + "loss": 4.120477387914434e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 765, + "tokens/total": 11549908, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 247312 + }, + { + "epoch": 11.96875, + "grad_norm": 0.00738960737362504, + "learning_rate": 7.659228375069827e-05, + "loss": 0.00011856225319206715, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 766, + "tokens/total": 11565044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 247636 + }, + { + "epoch": 11.984375, + "grad_norm": 0.012207427993416786, + "learning_rate": 7.652851762910505e-05, + "loss": 0.0002129970962414518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00021, + "step": 767, + "tokens/total": 11580188, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 247959 + }, + { + "epoch": 12.0, + "grad_norm": 0.005771435797214508, + "learning_rate": 7.646469539908316e-05, + "loss": 0.0001359613670501858, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 768, + "tokens/total": 11595328, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 248280 + }, + { + "epoch": 12.015625, + "grad_norm": 0.16021107137203217, + "learning_rate": 7.640081722696851e-05, + "loss": 0.002986999461427331, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00299, + "step": 769, + "tokens/total": 11610408, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 248603 + }, + { + "epoch": 12.03125, + "grad_norm": 0.06659352034330368, + "learning_rate": 7.633688327924279e-05, + "loss": 0.00023177383991423994, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00023, + "step": 770, + "tokens/total": 11625440, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 248929 + }, + { + "epoch": 12.046875, + "grad_norm": 0.043171677738428116, + "learning_rate": 7.627289372253304e-05, + "loss": 0.0004853203718084842, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00049, + "step": 771, + "tokens/total": 11640488, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 249250 + }, + { + "epoch": 12.0625, + "grad_norm": 0.06471310555934906, + "learning_rate": 7.620884872361122e-05, + "loss": 0.00033691420685499907, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00034, + "step": 772, + "tokens/total": 11655628, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 249573 + }, + { + "epoch": 12.078125, + "grad_norm": 0.0012351584155112505, + "learning_rate": 7.61447484493938e-05, + "loss": 4.9392016080673784e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 773, + "tokens/total": 11670680, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 249895 + }, + { + "epoch": 12.09375, + "grad_norm": 0.012346632778644562, + "learning_rate": 7.608059306694137e-05, + "loss": 0.00012409746705088764, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00012, + "step": 774, + "tokens/total": 11685696, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 250222 + }, + { + "epoch": 12.109375, + "grad_norm": 0.0019148180726915598, + "learning_rate": 7.601638274345803e-05, + "loss": 7.570564048364758e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 775, + "tokens/total": 11700828, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 250546 + }, + { + "epoch": 12.125, + "grad_norm": 0.011315557174384594, + "learning_rate": 7.595211764629117e-05, + "loss": 7.948732672957703e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 776, + "tokens/total": 11715968, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 250869 + }, + { + "epoch": 12.140625, + "grad_norm": 0.0014873143518343568, + "learning_rate": 7.588779794293086e-05, + "loss": 5.8105400967178866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 777, + "tokens/total": 11731076, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 251195 + }, + { + "epoch": 12.15625, + "grad_norm": 0.0056472280994057655, + "learning_rate": 7.582342380100953e-05, + "loss": 5.892526314710267e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 778, + "tokens/total": 11746184, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 251521 + }, + { + "epoch": 12.171875, + "grad_norm": 0.007838607765734196, + "learning_rate": 7.575899538830149e-05, + "loss": 0.00010413450945634395, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 779, + "tokens/total": 11761240, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 251840 + }, + { + "epoch": 12.1875, + "grad_norm": 0.046443358063697815, + "learning_rate": 7.569451287272247e-05, + "loss": 0.0003978235472459346, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0004, + "step": 780, + "tokens/total": 11776172, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 252166 + }, + { + "epoch": 12.203125, + "grad_norm": 0.001991559285670519, + "learning_rate": 7.562997642232921e-05, + "loss": 7.929989078547806e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 781, + "tokens/total": 11791252, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 252493 + }, + { + "epoch": 12.21875, + "grad_norm": 0.0033611422404646873, + "learning_rate": 7.556538620531904e-05, + "loss": 7.862138591008261e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 782, + "tokens/total": 11806280, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 252815 + }, + { + "epoch": 12.234375, + "grad_norm": 0.002292010700330138, + "learning_rate": 7.55007423900294e-05, + "loss": 7.994483166839927e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00008, + "step": 783, + "tokens/total": 11821296, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 253138 + }, + { + "epoch": 12.25, + "grad_norm": 0.0009671649895608425, + "learning_rate": 7.54360451449374e-05, + "loss": 4.199397881166078e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 784, + "tokens/total": 11836420, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 253459 + }, + { + "epoch": 12.265625, + "grad_norm": 0.0023980247788131237, + "learning_rate": 7.537129463865945e-05, + "loss": 7.811621617292985e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00008, + "step": 785, + "tokens/total": 11851556, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 253785 + }, + { + "epoch": 12.28125, + "grad_norm": 0.005567625630646944, + "learning_rate": 7.530649103995074e-05, + "loss": 0.00014928210293874145, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00015, + "step": 786, + "tokens/total": 11866676, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 254111 + }, + { + "epoch": 12.296875, + "grad_norm": 0.005312805995345116, + "learning_rate": 7.524163451770481e-05, + "loss": 0.00014692469267174602, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00015, + "step": 787, + "tokens/total": 11881856, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 254433 + }, + { + "epoch": 12.3125, + "grad_norm": 0.005031264387071133, + "learning_rate": 7.517672524095319e-05, + "loss": 8.916858496377245e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 788, + "tokens/total": 11897048, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 254756 + }, + { + "epoch": 12.328125, + "grad_norm": 0.0007357180002145469, + "learning_rate": 7.511176337886485e-05, + "loss": 3.350479164510034e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00003, + "step": 789, + "tokens/total": 11912072, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 255074 + }, + { + "epoch": 12.34375, + "grad_norm": 0.0009413110092282295, + "learning_rate": 7.50467491007458e-05, + "loss": 3.6423676647245884e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 790, + "tokens/total": 11927120, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 255399 + }, + { + "epoch": 12.359375, + "grad_norm": 0.0008683132473379374, + "learning_rate": 7.498168257603875e-05, + "loss": 3.2453805033583194e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 791, + "tokens/total": 11942172, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 255723 + }, + { + "epoch": 12.375, + "grad_norm": 0.001063066185452044, + "learning_rate": 7.491656397432247e-05, + "loss": 2.9047278076177463e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 792, + "tokens/total": 11957288, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 256046 + }, + { + "epoch": 12.390625, + "grad_norm": 0.0013455420266836882, + "learning_rate": 7.485139346531149e-05, + "loss": 4.68404614366591e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 793, + "tokens/total": 11972424, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 256368 + }, + { + "epoch": 12.40625, + "grad_norm": 0.0037693495396524668, + "learning_rate": 7.478617121885563e-05, + "loss": 6.180455966386944e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 794, + "tokens/total": 11987508, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 256692 + }, + { + "epoch": 12.421875, + "grad_norm": 0.0008823846001178026, + "learning_rate": 7.472089740493959e-05, + "loss": 3.486638888716698e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 795, + "tokens/total": 12002564, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 257012 + }, + { + "epoch": 12.4375, + "grad_norm": 0.0025711311027407646, + "learning_rate": 7.46555721936824e-05, + "loss": 6.127560482127592e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 796, + "tokens/total": 12017704, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 257339 + }, + { + "epoch": 12.453125, + "grad_norm": 0.020793205127120018, + "learning_rate": 7.459019575533707e-05, + "loss": 5.980060814181343e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 797, + "tokens/total": 12032856, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 257664 + }, + { + "epoch": 12.46875, + "grad_norm": 0.0010502856457605958, + "learning_rate": 7.452476826029011e-05, + "loss": 3.661629307316616e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00004, + "step": 798, + "tokens/total": 12048028, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 257988 + }, + { + "epoch": 12.484375, + "grad_norm": 0.0005576885305345058, + "learning_rate": 7.445928987906114e-05, + "loss": 2.2148260541143827e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00002, + "step": 799, + "tokens/total": 12063092, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 258315 + }, + { + "epoch": 12.5, + "grad_norm": 0.0005081727867946029, + "learning_rate": 7.439376078230234e-05, + "loss": 2.0269433662178926e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 800, + "tokens/total": 12078208, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 258638 + }, + { + "epoch": 12.515625, + "grad_norm": 0.0006259866640903056, + "learning_rate": 7.432818114079813e-05, + "loss": 2.4334258341696113e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 801, + "tokens/total": 12093348, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 258963 + }, + { + "epoch": 12.53125, + "grad_norm": 0.0003684913972392678, + "learning_rate": 7.426255112546458e-05, + "loss": 1.6602209143457003e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 802, + "tokens/total": 12108412, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 259287 + }, + { + "epoch": 12.546875, + "grad_norm": 0.0004341568856034428, + "learning_rate": 7.419687090734912e-05, + "loss": 1.815272480598651e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 803, + "tokens/total": 12123480, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 259607 + }, + { + "epoch": 12.5625, + "grad_norm": 0.008828886784613132, + "learning_rate": 7.413114065762999e-05, + "loss": 7.399358582915738e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00007, + "step": 804, + "tokens/total": 12138644, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 259933 + }, + { + "epoch": 12.578125, + "grad_norm": 0.0007751079392619431, + "learning_rate": 7.406536054761584e-05, + "loss": 1.663630609982647e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 805, + "tokens/total": 12153664, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 260256 + }, + { + "epoch": 12.59375, + "grad_norm": 0.00035115418722853065, + "learning_rate": 7.399953074874523e-05, + "loss": 1.597570371814072e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 806, + "tokens/total": 12168872, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 260582 + }, + { + "epoch": 12.609375, + "grad_norm": 0.0002732598513830453, + "learning_rate": 7.393365143258626e-05, + "loss": 1.3219183529145084e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 807, + "tokens/total": 12184012, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 260909 + }, + { + "epoch": 12.625, + "grad_norm": 0.0003034104302059859, + "learning_rate": 7.386772277083606e-05, + "loss": 1.4620679394283798e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 808, + "tokens/total": 12198968, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 261237 + }, + { + "epoch": 12.640625, + "grad_norm": 0.00048796235932968557, + "learning_rate": 7.38017449353204e-05, + "loss": 1.7275449863518588e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 809, + "tokens/total": 12214048, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 261560 + }, + { + "epoch": 12.65625, + "grad_norm": 0.0005510294577106833, + "learning_rate": 7.373571809799319e-05, + "loss": 1.772215728124138e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 810, + "tokens/total": 12229144, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 261882 + }, + { + "epoch": 12.671875, + "grad_norm": 0.0004420862242113799, + "learning_rate": 7.3669642430936e-05, + "loss": 1.773734766175039e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 811, + "tokens/total": 12244368, + "tokens/train_per_sec_per_gpu": 8.85, + "tokens/trainable": 262199 + }, + { + "epoch": 12.6875, + "grad_norm": 0.00022525829263031483, + "learning_rate": 7.360351810635774e-05, + "loss": 1.1139311936858576e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 812, + "tokens/total": 12259492, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 262525 + }, + { + "epoch": 12.703125, + "grad_norm": 0.00035630102502182126, + "learning_rate": 7.353734529659409e-05, + "loss": 1.4786606698180549e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 813, + "tokens/total": 12274608, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 262850 + }, + { + "epoch": 12.71875, + "grad_norm": 0.0002704367507249117, + "learning_rate": 7.347112417410712e-05, + "loss": 1.203708779939916e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 814, + "tokens/total": 12289780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 263171 + }, + { + "epoch": 12.734375, + "grad_norm": 0.0003859064891003072, + "learning_rate": 7.340485491148478e-05, + "loss": 1.5924519175314344e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 815, + "tokens/total": 12304880, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 263495 + }, + { + "epoch": 12.75, + "grad_norm": 0.00031393321114592254, + "learning_rate": 7.33385376814405e-05, + "loss": 1.391779187542852e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 816, + "tokens/total": 12319984, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 263818 + }, + { + "epoch": 12.765625, + "grad_norm": 0.0002688078675419092, + "learning_rate": 7.327217265681276e-05, + "loss": 1.0123962056240998e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 817, + "tokens/total": 12335172, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 264139 + }, + { + "epoch": 12.78125, + "grad_norm": 0.00039067966281436384, + "learning_rate": 7.320576001056452e-05, + "loss": 1.2487973435781896e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 818, + "tokens/total": 12350312, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 264466 + }, + { + "epoch": 12.796875, + "grad_norm": 0.0007766951457597315, + "learning_rate": 7.313929991578294e-05, + "loss": 2.187816789955832e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 819, + "tokens/total": 12365496, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 264789 + }, + { + "epoch": 12.8125, + "grad_norm": 0.004195546731352806, + "learning_rate": 7.307279254567882e-05, + "loss": 2.1917332560406066e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 820, + "tokens/total": 12380560, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 265112 + }, + { + "epoch": 12.828125, + "grad_norm": 0.000822481291834265, + "learning_rate": 7.300623807358614e-05, + "loss": 1.3468166798702441e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 821, + "tokens/total": 12395512, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 265435 + }, + { + "epoch": 12.84375, + "grad_norm": 0.0008303842041641474, + "learning_rate": 7.293963667296164e-05, + "loss": 1.9158533177687787e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 822, + "tokens/total": 12410528, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 265759 + }, + { + "epoch": 12.859375, + "grad_norm": 0.0003144189831800759, + "learning_rate": 7.287298851738442e-05, + "loss": 1.301559132116381e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 823, + "tokens/total": 12425616, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 266081 + }, + { + "epoch": 12.875, + "grad_norm": 0.00048050557961687446, + "learning_rate": 7.280629378055536e-05, + "loss": 8.131032700475771e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 824, + "tokens/total": 12440816, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 266400 + }, + { + "epoch": 12.890625, + "grad_norm": 0.00019102991791442037, + "learning_rate": 7.273955263629683e-05, + "loss": 9.445895557291806e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 825, + "tokens/total": 12455896, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 266720 + }, + { + "epoch": 12.90625, + "grad_norm": 0.00016436811711173505, + "learning_rate": 7.267276525855204e-05, + "loss": 7.868907232477795e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 826, + "tokens/total": 12470980, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 267036 + }, + { + "epoch": 12.921875, + "grad_norm": 0.00024990460951812565, + "learning_rate": 7.260593182138478e-05, + "loss": 1.1091404303442687e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 827, + "tokens/total": 12486052, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 267361 + }, + { + "epoch": 12.9375, + "grad_norm": 0.00034195027546957135, + "learning_rate": 7.253905249897888e-05, + "loss": 1.4048514458409045e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 828, + "tokens/total": 12501208, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 267688 + }, + { + "epoch": 12.953125, + "grad_norm": 0.0006922264001332223, + "learning_rate": 7.24721274656377e-05, + "loss": 1.8679204004001804e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 829, + "tokens/total": 12516164, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 268009 + }, + { + "epoch": 12.96875, + "grad_norm": 0.0006072817486710846, + "learning_rate": 7.240515689578378e-05, + "loss": 1.2583751413330901e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 830, + "tokens/total": 12531236, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 268332 + }, + { + "epoch": 12.984375, + "grad_norm": 0.00036045577144250274, + "learning_rate": 7.233814096395834e-05, + "loss": 1.219031310029095e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 831, + "tokens/total": 12546304, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 268654 + }, + { + "epoch": 13.0, + "grad_norm": 0.0004655960074160248, + "learning_rate": 7.22710798448208e-05, + "loss": 1.2151538612670265e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 832, + "tokens/total": 12561532, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 268970 + }, + { + "epoch": 13.015625, + "grad_norm": 0.0011391107691451907, + "learning_rate": 7.220397371314832e-05, + "loss": 9.716892236610875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 833, + "tokens/total": 12576708, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 269295 + }, + { + "epoch": 13.03125, + "grad_norm": 0.0003668550925794989, + "learning_rate": 7.213682274383549e-05, + "loss": 1.3395142559602391e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 834, + "tokens/total": 12591696, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 269614 + }, + { + "epoch": 13.046875, + "grad_norm": 0.00019505902309902012, + "learning_rate": 7.206962711189361e-05, + "loss": 8.56197948451154e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 835, + "tokens/total": 12606712, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 269937 + }, + { + "epoch": 13.0625, + "grad_norm": 0.00018391333287581801, + "learning_rate": 7.20023869924505e-05, + "loss": 9.066057828022167e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 836, + "tokens/total": 12621748, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 270258 + }, + { + "epoch": 13.078125, + "grad_norm": 0.00025342011940665543, + "learning_rate": 7.193510256074983e-05, + "loss": 9.309092092735227e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 837, + "tokens/total": 12636812, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 270586 + }, + { + "epoch": 13.09375, + "grad_norm": 0.00015372691268566996, + "learning_rate": 7.186777399215087e-05, + "loss": 7.696884495089762e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 838, + "tokens/total": 12651848, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 270908 + }, + { + "epoch": 13.109375, + "grad_norm": 0.0002597416169010103, + "learning_rate": 7.18004014621278e-05, + "loss": 1.1081143384217285e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 839, + "tokens/total": 12666920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 271225 + }, + { + "epoch": 13.125, + "grad_norm": 0.000603168155066669, + "learning_rate": 7.173298514626945e-05, + "loss": 1.642096322029829e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 840, + "tokens/total": 12682052, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 271551 + }, + { + "epoch": 13.140625, + "grad_norm": 0.00027083809254691005, + "learning_rate": 7.166552522027874e-05, + "loss": 1.038958453136729e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 841, + "tokens/total": 12697224, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 271879 + }, + { + "epoch": 13.15625, + "grad_norm": 0.00020748525275848806, + "learning_rate": 7.159802185997228e-05, + "loss": 9.541769941279199e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 842, + "tokens/total": 12712252, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 272200 + }, + { + "epoch": 13.171875, + "grad_norm": 0.0002647298970259726, + "learning_rate": 7.153047524127982e-05, + "loss": 8.699573299963959e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 843, + "tokens/total": 12727476, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 272523 + }, + { + "epoch": 13.1875, + "grad_norm": 0.00018993821868207306, + "learning_rate": 7.146288554024389e-05, + "loss": 8.366517249669414e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 844, + "tokens/total": 12742488, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 272843 + }, + { + "epoch": 13.203125, + "grad_norm": 0.00023091072216629982, + "learning_rate": 7.139525293301933e-05, + "loss": 9.51737820287235e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 845, + "tokens/total": 12757500, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 273165 + }, + { + "epoch": 13.21875, + "grad_norm": 0.0001466372632421553, + "learning_rate": 7.132757759587276e-05, + "loss": 7.0185815275181085e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 846, + "tokens/total": 12772632, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 273489 + }, + { + "epoch": 13.234375, + "grad_norm": 0.00025577039923518896, + "learning_rate": 7.125985970518216e-05, + "loss": 8.753009751671925e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 847, + "tokens/total": 12787728, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 273814 + }, + { + "epoch": 13.25, + "grad_norm": 0.0004935560864396393, + "learning_rate": 7.119209943743645e-05, + "loss": 1.4408618881134316e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 848, + "tokens/total": 12802904, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 274141 + }, + { + "epoch": 13.265625, + "grad_norm": 0.00019123018137179315, + "learning_rate": 7.112429696923496e-05, + "loss": 8.022295332921203e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 849, + "tokens/total": 12818004, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 274467 + }, + { + "epoch": 13.28125, + "grad_norm": 0.00016225746367126703, + "learning_rate": 7.105645247728703e-05, + "loss": 7.617646133439848e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 850, + "tokens/total": 12833000, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 274788 + }, + { + "epoch": 13.296875, + "grad_norm": 0.0003442743036430329, + "learning_rate": 7.098856613841153e-05, + "loss": 1.0638362255122047e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 851, + "tokens/total": 12848060, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 275107 + }, + { + "epoch": 13.3125, + "grad_norm": 0.00021005570306442678, + "learning_rate": 7.092063812953636e-05, + "loss": 9.465689799981192e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 852, + "tokens/total": 12863264, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 275429 + }, + { + "epoch": 13.328125, + "grad_norm": 0.0003693629114422947, + "learning_rate": 7.085266862769804e-05, + "loss": 9.904328180709854e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 853, + "tokens/total": 12878476, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 275754 + }, + { + "epoch": 13.34375, + "grad_norm": 0.0003127990639768541, + "learning_rate": 7.078465781004124e-05, + "loss": 9.77234412857797e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 854, + "tokens/total": 12893452, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 276074 + }, + { + "epoch": 13.359375, + "grad_norm": 0.0002485750592313707, + "learning_rate": 7.071660585381828e-05, + "loss": 9.048808351508342e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 855, + "tokens/total": 12908556, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 276397 + }, + { + "epoch": 13.375, + "grad_norm": 0.00015115922724362463, + "learning_rate": 7.064851293638875e-05, + "loss": 6.688484972983133e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 856, + "tokens/total": 12923656, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 276721 + }, + { + "epoch": 13.390625, + "grad_norm": 0.00015193774015642703, + "learning_rate": 7.05803792352189e-05, + "loss": 6.37586708762683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 857, + "tokens/total": 12938808, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 277050 + }, + { + "epoch": 13.40625, + "grad_norm": 0.000173087595612742, + "learning_rate": 7.05122049278814e-05, + "loss": 7.450259545294102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 858, + "tokens/total": 12954044, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 277379 + }, + { + "epoch": 13.421875, + "grad_norm": 0.00018124426424037665, + "learning_rate": 7.044399019205467e-05, + "loss": 8.668101145303808e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 859, + "tokens/total": 12969096, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 277701 + }, + { + "epoch": 13.4375, + "grad_norm": 0.00021906101028434932, + "learning_rate": 7.037573520552248e-05, + "loss": 8.308878932439256e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 860, + "tokens/total": 12984184, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 278019 + }, + { + "epoch": 13.453125, + "grad_norm": 0.0005401084199547768, + "learning_rate": 7.030744014617355e-05, + "loss": 1.5149706086958759e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 861, + "tokens/total": 12999240, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 278340 + }, + { + "epoch": 13.46875, + "grad_norm": 0.00013522176595870405, + "learning_rate": 7.023910519200101e-05, + "loss": 5.80459618504392e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 862, + "tokens/total": 13014336, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 278662 + }, + { + "epoch": 13.484375, + "grad_norm": 0.0001677647087490186, + "learning_rate": 7.017073052110199e-05, + "loss": 7.4704535109049175e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 863, + "tokens/total": 13029416, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 278986 + }, + { + "epoch": 13.5, + "grad_norm": 0.0003589221742004156, + "learning_rate": 7.01023163116771e-05, + "loss": 1.0307087904948276e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 864, + "tokens/total": 13044640, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 279311 + }, + { + "epoch": 13.515625, + "grad_norm": 0.00018898042617365718, + "learning_rate": 7.003386274203004e-05, + "loss": 8.296982741740067e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 865, + "tokens/total": 13059684, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 279634 + }, + { + "epoch": 13.53125, + "grad_norm": 0.0010237118694931269, + "learning_rate": 6.996536999056702e-05, + "loss": 2.2999636712484062e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 866, + "tokens/total": 13074760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 279960 + }, + { + "epoch": 13.546875, + "grad_norm": 0.00014065396680962294, + "learning_rate": 6.989683823579647e-05, + "loss": 5.4438582992588636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 867, + "tokens/total": 13089980, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 280284 + }, + { + "epoch": 13.5625, + "grad_norm": 0.00012837437679991126, + "learning_rate": 6.982826765632837e-05, + "loss": 6.230836788745364e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 868, + "tokens/total": 13104960, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 280607 + }, + { + "epoch": 13.578125, + "grad_norm": 0.0008848016150295734, + "learning_rate": 6.975965843087396e-05, + "loss": 7.680561793677043e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 869, + "tokens/total": 13119920, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 280927 + }, + { + "epoch": 13.59375, + "grad_norm": 0.0001804027851903811, + "learning_rate": 6.969101073824514e-05, + "loss": 7.143568836909253e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 870, + "tokens/total": 13135080, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 281246 + }, + { + "epoch": 13.609375, + "grad_norm": 0.00018121872562915087, + "learning_rate": 6.962232475735414e-05, + "loss": 7.786889000271913e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 871, + "tokens/total": 13150184, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 281569 + }, + { + "epoch": 13.625, + "grad_norm": 0.00016253715148195624, + "learning_rate": 6.95536006672129e-05, + "loss": 5.7847191783366725e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 872, + "tokens/total": 13165276, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 281892 + }, + { + "epoch": 13.640625, + "grad_norm": 0.00018668230040930212, + "learning_rate": 6.948483864693274e-05, + "loss": 6.778848728572484e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 873, + "tokens/total": 13180380, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 282209 + }, + { + "epoch": 13.65625, + "grad_norm": 0.00014120223931968212, + "learning_rate": 6.941603887572378e-05, + "loss": 5.9225098993920255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 874, + "tokens/total": 13195420, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 282528 + }, + { + "epoch": 13.671875, + "grad_norm": 0.00010363444744143635, + "learning_rate": 6.93472015328946e-05, + "loss": 4.5792908167641144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 875, + "tokens/total": 13210508, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 282852 + }, + { + "epoch": 13.6875, + "grad_norm": 0.0001297659910051152, + "learning_rate": 6.927832679785162e-05, + "loss": 6.144676717667608e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 876, + "tokens/total": 13225704, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 283177 + }, + { + "epoch": 13.703125, + "grad_norm": 0.00011152560909977183, + "learning_rate": 6.92094148500988e-05, + "loss": 5.04549916513497e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 877, + "tokens/total": 13240808, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 283502 + }, + { + "epoch": 13.71875, + "grad_norm": 0.0006136860465630889, + "learning_rate": 6.914046586923699e-05, + "loss": 1.2460717698559165e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 878, + "tokens/total": 13255932, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 283826 + }, + { + "epoch": 13.734375, + "grad_norm": 0.00021741165255662054, + "learning_rate": 6.907148003496365e-05, + "loss": 8.354354577022605e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 879, + "tokens/total": 13271004, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 284153 + }, + { + "epoch": 13.75, + "grad_norm": 0.00019565780530683696, + "learning_rate": 6.900245752707218e-05, + "loss": 7.582655598525889e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 880, + "tokens/total": 13286120, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 284479 + }, + { + "epoch": 13.765625, + "grad_norm": 0.00013492730795405805, + "learning_rate": 6.893339852545169e-05, + "loss": 5.292617061058991e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 881, + "tokens/total": 13301260, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 284804 + }, + { + "epoch": 13.78125, + "grad_norm": 0.00023065209097694606, + "learning_rate": 6.886430321008632e-05, + "loss": 8.284407158498652e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 882, + "tokens/total": 13316384, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 285126 + }, + { + "epoch": 13.796875, + "grad_norm": 0.00034141886862926185, + "learning_rate": 6.879517176105485e-05, + "loss": 8.606583833170589e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 883, + "tokens/total": 13331492, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 285449 + }, + { + "epoch": 13.8125, + "grad_norm": 0.0001509837165940553, + "learning_rate": 6.872600435853025e-05, + "loss": 6.6483621594670694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 884, + "tokens/total": 13346580, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 285770 + }, + { + "epoch": 13.828125, + "grad_norm": 0.00012014619278488681, + "learning_rate": 6.865680118277922e-05, + "loss": 4.938416168442927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 885, + "tokens/total": 13361648, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 286101 + }, + { + "epoch": 13.84375, + "grad_norm": 0.0001542358222650364, + "learning_rate": 6.858756241416163e-05, + "loss": 6.173019755806308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 886, + "tokens/total": 13376720, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 286424 + }, + { + "epoch": 13.859375, + "grad_norm": 0.00029009420541115105, + "learning_rate": 6.85182882331302e-05, + "loss": 8.017850632313639e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 887, + "tokens/total": 13391656, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 286745 + }, + { + "epoch": 13.875, + "grad_norm": 0.00021704383834730834, + "learning_rate": 6.844897882022984e-05, + "loss": 8.939020517573226e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 888, + "tokens/total": 13406776, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 287068 + }, + { + "epoch": 13.890625, + "grad_norm": 0.00013858875900041312, + "learning_rate": 6.837963435609737e-05, + "loss": 6.301908342720708e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 889, + "tokens/total": 13421912, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 287394 + }, + { + "epoch": 13.90625, + "grad_norm": 0.00012034752580802888, + "learning_rate": 6.831025502146093e-05, + "loss": 5.209335995459696e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 890, + "tokens/total": 13436940, + "tokens/train_per_sec_per_gpu": 9.93, + "tokens/trainable": 287718 + }, + { + "epoch": 13.921875, + "grad_norm": 0.0016086268005892634, + "learning_rate": 6.824084099713952e-05, + "loss": 1.0298252163920552e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 891, + "tokens/total": 13452080, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 288049 + }, + { + "epoch": 13.9375, + "grad_norm": 0.0001797742152120918, + "learning_rate": 6.817139246404257e-05, + "loss": 6.938968908798415e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 892, + "tokens/total": 13467180, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 288368 + }, + { + "epoch": 13.953125, + "grad_norm": 0.00013181679241824895, + "learning_rate": 6.810190960316948e-05, + "loss": 4.9134459914057516e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 893, + "tokens/total": 13482280, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 288691 + }, + { + "epoch": 13.96875, + "grad_norm": 0.00016014109132811427, + "learning_rate": 6.803239259560903e-05, + "loss": 6.6439038164389785e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 894, + "tokens/total": 13497352, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 289015 + }, + { + "epoch": 13.984375, + "grad_norm": 0.0005348578561097383, + "learning_rate": 6.796284162253911e-05, + "loss": 1.158072154794354e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 895, + "tokens/total": 13512400, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 289336 + }, + { + "epoch": 14.0, + "grad_norm": 0.00029467378044500947, + "learning_rate": 6.789325686522602e-05, + "loss": 7.453762464137981e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 896, + "tokens/total": 13527548, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 289660 + }, + { + "epoch": 14.015625, + "grad_norm": 0.00011753098806366324, + "learning_rate": 6.782363850502417e-05, + "loss": 4.7445500968024135e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 897, + "tokens/total": 13542724, + "tokens/train_per_sec_per_gpu": 9.0, + "tokens/trainable": 289984 + }, + { + "epoch": 14.03125, + "grad_norm": 0.00015148636884987354, + "learning_rate": 6.775398672337556e-05, + "loss": 5.323068307916401e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 898, + "tokens/total": 13557752, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 290306 + }, + { + "epoch": 14.046875, + "grad_norm": 0.00016822446195874363, + "learning_rate": 6.768430170180924e-05, + "loss": 6.400650818250142e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 899, + "tokens/total": 13572924, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 290629 + }, + { + "epoch": 14.0625, + "grad_norm": 0.00011592747614486143, + "learning_rate": 6.761458362194097e-05, + "loss": 4.885735052084783e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 900, + "tokens/total": 13588024, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 290959 + }, + { + "epoch": 14.078125, + "grad_norm": 0.00014252217079047114, + "learning_rate": 6.754483266547253e-05, + "loss": 5.938222329859855e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 901, + "tokens/total": 13603244, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 291284 + }, + { + "epoch": 14.09375, + "grad_norm": 0.00022220492246560752, + "learning_rate": 6.747504901419158e-05, + "loss": 7.164710041251965e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00001, + "step": 902, + "tokens/total": 13618464, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 291607 + }, + { + "epoch": 14.109375, + "grad_norm": 0.0002508973120711744, + "learning_rate": 6.740523284997083e-05, + "loss": 9.060651791514829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 903, + "tokens/total": 13633456, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 291932 + }, + { + "epoch": 14.125, + "grad_norm": 0.00014073106285650283, + "learning_rate": 6.733538435476779e-05, + "loss": 6.454753474827157e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 904, + "tokens/total": 13648544, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 292255 + }, + { + "epoch": 14.140625, + "grad_norm": 0.00011323496437398717, + "learning_rate": 6.726550371062418e-05, + "loss": 4.646141860575881e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 905, + "tokens/total": 13663696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 292580 + }, + { + "epoch": 14.15625, + "grad_norm": 0.0005007471772842109, + "learning_rate": 6.719559109966563e-05, + "loss": 6.985198069742182e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 906, + "tokens/total": 13678704, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 292902 + }, + { + "epoch": 14.171875, + "grad_norm": 0.00013906758977100253, + "learning_rate": 6.712564670410095e-05, + "loss": 5.2512505135382526e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 907, + "tokens/total": 13693768, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 293226 + }, + { + "epoch": 14.1875, + "grad_norm": 0.0005896158399991691, + "learning_rate": 6.705567070622186e-05, + "loss": 7.144201845221687e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 908, + "tokens/total": 13708908, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 293551 + }, + { + "epoch": 14.203125, + "grad_norm": 0.00013230345211923122, + "learning_rate": 6.698566328840242e-05, + "loss": 5.474795216287021e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 909, + "tokens/total": 13724060, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 293875 + }, + { + "epoch": 14.21875, + "grad_norm": 0.0001060825161403045, + "learning_rate": 6.69156246330986e-05, + "loss": 4.551543497655075e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 910, + "tokens/total": 13739192, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 294200 + }, + { + "epoch": 14.234375, + "grad_norm": 0.0001285890903091058, + "learning_rate": 6.684555492284775e-05, + "loss": 4.3470263335620984e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 911, + "tokens/total": 13754336, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 294526 + }, + { + "epoch": 14.25, + "grad_norm": 0.0001352222025161609, + "learning_rate": 6.677545434026816e-05, + "loss": 5.622450316877803e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 912, + "tokens/total": 13769412, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 294850 + }, + { + "epoch": 14.265625, + "grad_norm": 0.0001278308336623013, + "learning_rate": 6.670532306805861e-05, + "loss": 5.480685558723053e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 913, + "tokens/total": 13784544, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 295174 + }, + { + "epoch": 14.28125, + "grad_norm": 0.00010411172843305394, + "learning_rate": 6.663516128899785e-05, + "loss": 4.7005287342472e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 914, + "tokens/total": 13799696, + "tokens/train_per_sec_per_gpu": 8.64, + "tokens/trainable": 295496 + }, + { + "epoch": 14.296875, + "grad_norm": 0.00011363694648025557, + "learning_rate": 6.656496918594413e-05, + "loss": 4.781138159160037e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 915, + "tokens/total": 13814812, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 295825 + }, + { + "epoch": 14.3125, + "grad_norm": 0.00011337734031258151, + "learning_rate": 6.649474694183474e-05, + "loss": 4.443406851351028e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 916, + "tokens/total": 13829944, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 296147 + }, + { + "epoch": 14.328125, + "grad_norm": 0.0002905918227043003, + "learning_rate": 6.642449473968551e-05, + "loss": 8.787519618635997e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 917, + "tokens/total": 13845084, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 296470 + }, + { + "epoch": 14.34375, + "grad_norm": 0.00013545055116992444, + "learning_rate": 6.635421276259038e-05, + "loss": 5.458176474348875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 918, + "tokens/total": 13860172, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 296797 + }, + { + "epoch": 14.359375, + "grad_norm": 0.0001511846057837829, + "learning_rate": 6.628390119372085e-05, + "loss": 5.318036073731491e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 919, + "tokens/total": 13875296, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 297118 + }, + { + "epoch": 14.375, + "grad_norm": 0.00015693627938162535, + "learning_rate": 6.621356021632556e-05, + "loss": 5.894385594729101e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 920, + "tokens/total": 13890396, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 297441 + }, + { + "epoch": 14.390625, + "grad_norm": 0.0002013167686527595, + "learning_rate": 6.614319001372981e-05, + "loss": 6.947778729227139e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 921, + "tokens/total": 13905476, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 297763 + }, + { + "epoch": 14.40625, + "grad_norm": 0.00023332379350904375, + "learning_rate": 6.607279076933509e-05, + "loss": 5.532775048777694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 922, + "tokens/total": 13920536, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 298088 + }, + { + "epoch": 14.421875, + "grad_norm": 0.00020745137589983642, + "learning_rate": 6.600236266661848e-05, + "loss": 5.885206519451458e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 923, + "tokens/total": 13935580, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 298411 + }, + { + "epoch": 14.4375, + "grad_norm": 0.00011867156717926264, + "learning_rate": 6.593190588913239e-05, + "loss": 4.981664460501634e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 924, + "tokens/total": 13950700, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 298733 + }, + { + "epoch": 14.453125, + "grad_norm": 0.00014705830835737288, + "learning_rate": 6.586142062050387e-05, + "loss": 5.981057256576605e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 925, + "tokens/total": 13965820, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 299054 + }, + { + "epoch": 14.46875, + "grad_norm": 9.865839820122346e-05, + "learning_rate": 6.579090704443432e-05, + "loss": 4.12433792007505e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 926, + "tokens/total": 13980948, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 299375 + }, + { + "epoch": 14.484375, + "grad_norm": 9.338240488432348e-05, + "learning_rate": 6.572036534469879e-05, + "loss": 3.7122513276699465e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 927, + "tokens/total": 13996084, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 299700 + }, + { + "epoch": 14.5, + "grad_norm": 9.448266064282507e-05, + "learning_rate": 6.564979570514577e-05, + "loss": 4.0880463529902045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 928, + "tokens/total": 14011092, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 300022 + }, + { + "epoch": 14.515625, + "grad_norm": 0.00014782392827328295, + "learning_rate": 6.557919830969644e-05, + "loss": 4.351799361757003e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 929, + "tokens/total": 14026324, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 300348 + }, + { + "epoch": 14.53125, + "grad_norm": 0.00013573178148362786, + "learning_rate": 6.550857334234442e-05, + "loss": 5.508952199306805e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 930, + "tokens/total": 14041448, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 300671 + }, + { + "epoch": 14.546875, + "grad_norm": 0.0001272056979360059, + "learning_rate": 6.543792098715513e-05, + "loss": 4.981610345566878e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 931, + "tokens/total": 14056636, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 300994 + }, + { + "epoch": 14.5625, + "grad_norm": 0.00018646674288902432, + "learning_rate": 6.536724142826536e-05, + "loss": 4.6987652240204625e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 932, + "tokens/total": 14071748, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 301323 + }, + { + "epoch": 14.578125, + "grad_norm": 8.852640894474462e-05, + "learning_rate": 6.529653484988287e-05, + "loss": 3.6944829844287597e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 933, + "tokens/total": 14086812, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 301649 + }, + { + "epoch": 14.59375, + "grad_norm": 6.746128929080442e-05, + "learning_rate": 6.522580143628573e-05, + "loss": 3.1192773803923046e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 934, + "tokens/total": 14101904, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 301975 + }, + { + "epoch": 14.609375, + "grad_norm": 0.00010212919005425647, + "learning_rate": 6.515504137182206e-05, + "loss": 4.187063041172223e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 935, + "tokens/total": 14117012, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 302295 + }, + { + "epoch": 14.625, + "grad_norm": 7.991518214112148e-05, + "learning_rate": 6.508425484090935e-05, + "loss": 3.4409877116559073e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 936, + "tokens/total": 14131976, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 302617 + }, + { + "epoch": 14.640625, + "grad_norm": 0.00023990595946088433, + "learning_rate": 6.501344202803414e-05, + "loss": 6.183725417940877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 937, + "tokens/total": 14147008, + "tokens/train_per_sec_per_gpu": 8.96, + "tokens/trainable": 302937 + }, + { + "epoch": 14.65625, + "grad_norm": 0.0003819094563368708, + "learning_rate": 6.494260311775145e-05, + "loss": 7.558513061667327e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 938, + "tokens/total": 14162152, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 303265 + }, + { + "epoch": 14.671875, + "grad_norm": 0.00010216620285063982, + "learning_rate": 6.487173829468424e-05, + "loss": 4.277219886716921e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 939, + "tokens/total": 14177152, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 303589 + }, + { + "epoch": 14.6875, + "grad_norm": 0.00010409334936412051, + "learning_rate": 6.480084774352312e-05, + "loss": 4.4556345528690144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 940, + "tokens/total": 14192124, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 303911 + }, + { + "epoch": 14.703125, + "grad_norm": 0.0006500252056866884, + "learning_rate": 6.472993164902566e-05, + "loss": 1.3360363482206594e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 941, + "tokens/total": 14207176, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 304235 + }, + { + "epoch": 14.71875, + "grad_norm": 0.00026559430989436805, + "learning_rate": 6.465899019601607e-05, + "loss": 5.255849828245118e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 942, + "tokens/total": 14222308, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 304559 + }, + { + "epoch": 14.734375, + "grad_norm": 0.0003588323306757957, + "learning_rate": 6.458802356938462e-05, + "loss": 5.663363026542356e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 943, + "tokens/total": 14237520, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 304880 + }, + { + "epoch": 14.75, + "grad_norm": 0.000354075018549338, + "learning_rate": 6.451703195408714e-05, + "loss": 6.227821359061636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 944, + "tokens/total": 14252592, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 305200 + }, + { + "epoch": 14.765625, + "grad_norm": 7.481413194909692e-05, + "learning_rate": 6.444601553514472e-05, + "loss": 2.82400810647232e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 945, + "tokens/total": 14267684, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 305522 + }, + { + "epoch": 14.78125, + "grad_norm": 9.600890916772187e-05, + "learning_rate": 6.437497449764292e-05, + "loss": 3.910564373654779e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 946, + "tokens/total": 14282812, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 305844 + }, + { + "epoch": 14.796875, + "grad_norm": 0.00012574389984365553, + "learning_rate": 6.43039090267316e-05, + "loss": 4.624543180398177e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 947, + "tokens/total": 14298036, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 306167 + }, + { + "epoch": 14.8125, + "grad_norm": 0.0002485642035026103, + "learning_rate": 6.423281930762426e-05, + "loss": 6.751722139597405e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 948, + "tokens/total": 14313140, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 306489 + }, + { + "epoch": 14.828125, + "grad_norm": 0.00013306459004525095, + "learning_rate": 6.416170552559758e-05, + "loss": 3.5438904433249263e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 949, + "tokens/total": 14328408, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 306810 + }, + { + "epoch": 14.84375, + "grad_norm": 7.136823114706203e-05, + "learning_rate": 6.409056786599093e-05, + "loss": 3.3471510505478363e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 950, + "tokens/total": 14343448, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 307130 + }, + { + "epoch": 14.859375, + "grad_norm": 0.0002555957471486181, + "learning_rate": 6.401940651420598e-05, + "loss": 6.42076429357985e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 951, + "tokens/total": 14358560, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 307453 + }, + { + "epoch": 14.875, + "grad_norm": 0.000336786441039294, + "learning_rate": 6.39482216557061e-05, + "loss": 6.087861038395204e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 952, + "tokens/total": 14373584, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 307776 + }, + { + "epoch": 14.890625, + "grad_norm": 0.00014989046030677855, + "learning_rate": 6.387701347601593e-05, + "loss": 4.200231614959193e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 953, + "tokens/total": 14388648, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 308096 + }, + { + "epoch": 14.90625, + "grad_norm": 0.00010292268416378647, + "learning_rate": 6.380578216072088e-05, + "loss": 4.099169927940238e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 954, + "tokens/total": 14403620, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 308416 + }, + { + "epoch": 14.921875, + "grad_norm": 0.00010484905942576006, + "learning_rate": 6.373452789546669e-05, + "loss": 4.011551482108189e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 955, + "tokens/total": 14418724, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 308740 + }, + { + "epoch": 14.9375, + "grad_norm": 0.00016220823454204947, + "learning_rate": 6.36632508659589e-05, + "loss": 4.039769009978045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 956, + "tokens/total": 14433808, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 309062 + }, + { + "epoch": 14.953125, + "grad_norm": 7.400909817079082e-05, + "learning_rate": 6.359195125796233e-05, + "loss": 3.180170097039081e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 957, + "tokens/total": 14448956, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 309391 + }, + { + "epoch": 14.96875, + "grad_norm": 9.141374175669625e-05, + "learning_rate": 6.352062925730074e-05, + "loss": 3.640821205408429e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 958, + "tokens/total": 14463960, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 309716 + }, + { + "epoch": 14.984375, + "grad_norm": 0.00016905312077142298, + "learning_rate": 6.344928504985616e-05, + "loss": 3.3424146295146784e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 959, + "tokens/total": 14479012, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 310029 + }, + { + "epoch": 15.0, + "grad_norm": 6.48611894575879e-05, + "learning_rate": 6.337791882156857e-05, + "loss": 2.7029941520595457e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 960, + "tokens/total": 14494116, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 310350 + }, + { + "epoch": 15.015625, + "grad_norm": 0.0002493032079655677, + "learning_rate": 6.330653075843527e-05, + "loss": 6.20826813246822e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 961, + "tokens/total": 14509288, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 310676 + }, + { + "epoch": 15.03125, + "grad_norm": 7.431192352669314e-05, + "learning_rate": 6.323512104651052e-05, + "loss": 3.227911975045572e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 962, + "tokens/total": 14524308, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 310999 + }, + { + "epoch": 15.046875, + "grad_norm": 0.0001833956630434841, + "learning_rate": 6.316368987190498e-05, + "loss": 4.975226602255134e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 963, + "tokens/total": 14539320, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 311326 + }, + { + "epoch": 15.0625, + "grad_norm": 0.00022195035126060247, + "learning_rate": 6.309223742078527e-05, + "loss": 6.791829036956187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 964, + "tokens/total": 14554348, + "tokens/train_per_sec_per_gpu": 9.93, + "tokens/trainable": 311652 + }, + { + "epoch": 15.078125, + "grad_norm": 7.450357225025073e-05, + "learning_rate": 6.302076387937343e-05, + "loss": 3.1351062261819607e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 965, + "tokens/total": 14569396, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 311979 + }, + { + "epoch": 15.09375, + "grad_norm": 0.00014489535533357412, + "learning_rate": 6.294926943394646e-05, + "loss": 5.163768037164118e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 966, + "tokens/total": 14584500, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 312300 + }, + { + "epoch": 15.109375, + "grad_norm": 9.194896119879559e-05, + "learning_rate": 6.28777542708359e-05, + "loss": 3.1608269637217745e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 967, + "tokens/total": 14599592, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 312620 + }, + { + "epoch": 15.125, + "grad_norm": 9.030203364090994e-05, + "learning_rate": 6.280621857642722e-05, + "loss": 3.1681079235568177e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 968, + "tokens/total": 14614736, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 312944 + }, + { + "epoch": 15.140625, + "grad_norm": 6.451710942201316e-05, + "learning_rate": 6.273466253715941e-05, + "loss": 2.7942376163991867e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 969, + "tokens/total": 14629748, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 313271 + }, + { + "epoch": 15.15625, + "grad_norm": 6.718290387652814e-05, + "learning_rate": 6.266308633952454e-05, + "loss": 2.512852461222792e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 970, + "tokens/total": 14644736, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 313592 + }, + { + "epoch": 15.171875, + "grad_norm": 0.0002705650986172259, + "learning_rate": 6.259149017006713e-05, + "loss": 4.182214524917072e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 971, + "tokens/total": 14659816, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 313918 + }, + { + "epoch": 15.1875, + "grad_norm": 7.518421625718474e-05, + "learning_rate": 6.251987421538383e-05, + "loss": 3.160873802698916e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 972, + "tokens/total": 14674920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 314244 + }, + { + "epoch": 15.203125, + "grad_norm": 7.085589459165931e-05, + "learning_rate": 6.244823866212282e-05, + "loss": 2.948641622424475e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 973, + "tokens/total": 14690064, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 314568 + }, + { + "epoch": 15.21875, + "grad_norm": 0.00014159946294967085, + "learning_rate": 6.237658369698335e-05, + "loss": 4.136055395065341e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 974, + "tokens/total": 14705236, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 314888 + }, + { + "epoch": 15.234375, + "grad_norm": 0.0002387032873230055, + "learning_rate": 6.23049095067153e-05, + "loss": 4.655185875890311e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 975, + "tokens/total": 14720384, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 315217 + }, + { + "epoch": 15.25, + "grad_norm": 0.00033773118047975004, + "learning_rate": 6.223321627811857e-05, + "loss": 5.643796612275764e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 976, + "tokens/total": 14735432, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 315538 + }, + { + "epoch": 15.265625, + "grad_norm": 0.00013314354873728007, + "learning_rate": 6.216150419804279e-05, + "loss": 4.118748165637953e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 977, + "tokens/total": 14750520, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 315862 + }, + { + "epoch": 15.28125, + "grad_norm": 0.00013210996985435486, + "learning_rate": 6.208977345338665e-05, + "loss": 3.976341304223752e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 978, + "tokens/total": 14765648, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 316185 + }, + { + "epoch": 15.296875, + "grad_norm": 8.321408677147701e-05, + "learning_rate": 6.201802423109749e-05, + "loss": 3.2865746106836013e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 979, + "tokens/total": 14780724, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 316510 + }, + { + "epoch": 15.3125, + "grad_norm": 7.425444346154109e-05, + "learning_rate": 6.194625671817084e-05, + "loss": 3.290261247457238e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 980, + "tokens/total": 14795824, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 316830 + }, + { + "epoch": 15.328125, + "grad_norm": 9.601408964954317e-05, + "learning_rate": 6.187447110164986e-05, + "loss": 3.4851434520533076e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 981, + "tokens/total": 14810876, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 317153 + }, + { + "epoch": 15.34375, + "grad_norm": 0.00011997880937997252, + "learning_rate": 6.180266756862493e-05, + "loss": 4.0942386476672255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 982, + "tokens/total": 14825992, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 317473 + }, + { + "epoch": 15.359375, + "grad_norm": 9.007082553580403e-05, + "learning_rate": 6.173084630623311e-05, + "loss": 3.5502823720889864e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 983, + "tokens/total": 14840956, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 317796 + }, + { + "epoch": 15.375, + "grad_norm": 9.219747880706564e-05, + "learning_rate": 6.165900750165763e-05, + "loss": 3.0103624339972157e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 984, + "tokens/total": 14856184, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 318123 + }, + { + "epoch": 15.390625, + "grad_norm": 0.00012016884284093976, + "learning_rate": 6.15871513421275e-05, + "loss": 4.624607299774652e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 985, + "tokens/total": 14871248, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 318449 + }, + { + "epoch": 15.40625, + "grad_norm": 8.112434443319216e-05, + "learning_rate": 6.151527801491692e-05, + "loss": 3.1713218504592078e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 986, + "tokens/total": 14886400, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 318775 + }, + { + "epoch": 15.421875, + "grad_norm": 0.00018411068595014513, + "learning_rate": 6.144338770734487e-05, + "loss": 4.731994067697087e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 987, + "tokens/total": 14901536, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 319093 + }, + { + "epoch": 15.4375, + "grad_norm": 8.050377800827846e-05, + "learning_rate": 6.137148060677453e-05, + "loss": 2.952805061795516e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 988, + "tokens/total": 14916616, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 319418 + }, + { + "epoch": 15.453125, + "grad_norm": 0.0007351291715167463, + "learning_rate": 6.129955690061289e-05, + "loss": 3.0646751838503405e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 989, + "tokens/total": 14931756, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 319739 + }, + { + "epoch": 15.46875, + "grad_norm": 0.00018496651318855584, + "learning_rate": 6.122761677631022e-05, + "loss": 3.4498948480177205e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 990, + "tokens/total": 14946856, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 320062 + }, + { + "epoch": 15.484375, + "grad_norm": 9.208662959281355e-05, + "learning_rate": 6.115566042135953e-05, + "loss": 3.648818847068469e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 991, + "tokens/total": 14962100, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 320385 + }, + { + "epoch": 15.5, + "grad_norm": 7.000987534411252e-05, + "learning_rate": 6.10836880232962e-05, + "loss": 2.8676658985204995e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 992, + "tokens/total": 14977184, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 320706 + }, + { + "epoch": 15.515625, + "grad_norm": 0.00015829032054170966, + "learning_rate": 6.1011699769697384e-05, + "loss": 3.162227130815154e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 993, + "tokens/total": 14992300, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 321031 + }, + { + "epoch": 15.53125, + "grad_norm": 0.00022179844381753355, + "learning_rate": 6.0939695848181546e-05, + "loss": 4.528326371655567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 994, + "tokens/total": 15007460, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 321349 + }, + { + "epoch": 15.546875, + "grad_norm": 8.622360473964363e-05, + "learning_rate": 6.086767644640801e-05, + "loss": 3.1386755381390685e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 995, + "tokens/total": 15022532, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 321672 + }, + { + "epoch": 15.5625, + "grad_norm": 6.173367728479207e-05, + "learning_rate": 6.079564175207645e-05, + "loss": 2.665425427039736e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 996, + "tokens/total": 15037596, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 321995 + }, + { + "epoch": 15.578125, + "grad_norm": 6.35695832897909e-05, + "learning_rate": 6.072359195292635e-05, + "loss": 2.578557769084e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 997, + "tokens/total": 15052708, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 322320 + }, + { + "epoch": 15.59375, + "grad_norm": 6.942416075617075e-05, + "learning_rate": 6.0651527236736636e-05, + "loss": 2.850873443094315e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 998, + "tokens/total": 15067820, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 322642 + }, + { + "epoch": 15.609375, + "grad_norm": 9.018466516863555e-05, + "learning_rate": 6.057944779132505e-05, + "loss": 3.1195718293020036e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 999, + "tokens/total": 15082956, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 322968 + }, + { + "epoch": 15.625, + "grad_norm": 5.8539400924928486e-05, + "learning_rate": 6.050735380454774e-05, + "loss": 2.429555252092541e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1000, + "tokens/total": 15098020, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 323293 + }, + { + "epoch": 15.640625, + "grad_norm": 6.488087819889188e-05, + "learning_rate": 6.043524546429874e-05, + "loss": 2.5248300516977906e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1001, + "tokens/total": 15113016, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 323615 + }, + { + "epoch": 15.65625, + "grad_norm": 0.00016019836766645312, + "learning_rate": 6.036312295850954e-05, + "loss": 2.8190345346956747e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1002, + "tokens/total": 15128136, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 323945 + }, + { + "epoch": 15.671875, + "grad_norm": 0.0002470797044225037, + "learning_rate": 6.029098647514848e-05, + "loss": 4.359888407634571e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1003, + "tokens/total": 15143328, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 324273 + }, + { + "epoch": 15.6875, + "grad_norm": 5.69274379813578e-05, + "learning_rate": 6.0218836202220394e-05, + "loss": 2.493406100256834e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1004, + "tokens/total": 15158384, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 324598 + }, + { + "epoch": 15.703125, + "grad_norm": 6.755355570930988e-05, + "learning_rate": 6.014667232776601e-05, + "loss": 2.905426754296059e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1005, + "tokens/total": 15173492, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 324921 + }, + { + "epoch": 15.71875, + "grad_norm": 0.00014780242054257542, + "learning_rate": 6.0074495039861515e-05, + "loss": 4.1000862438522745e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1006, + "tokens/total": 15188732, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 325244 + }, + { + "epoch": 15.734375, + "grad_norm": 0.00014287748490460217, + "learning_rate": 6.000230452661806e-05, + "loss": 3.3332289604004472e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1007, + "tokens/total": 15203780, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 325567 + }, + { + "epoch": 15.75, + "grad_norm": 9.527063957648352e-05, + "learning_rate": 5.993010097618127e-05, + "loss": 3.591237828004523e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1008, + "tokens/total": 15218936, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 325892 + }, + { + "epoch": 15.765625, + "grad_norm": 7.01309836586006e-05, + "learning_rate": 5.985788457673074e-05, + "loss": 2.804142013701494e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1009, + "tokens/total": 15234040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 326212 + }, + { + "epoch": 15.78125, + "grad_norm": 0.00021756220667157322, + "learning_rate": 5.9785655516479535e-05, + "loss": 3.8052919535402907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1010, + "tokens/total": 15249068, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 326535 + }, + { + "epoch": 15.796875, + "grad_norm": 7.258858386194333e-05, + "learning_rate": 5.9713413983673774e-05, + "loss": 2.8980441584280925e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1011, + "tokens/total": 15264252, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 326858 + }, + { + "epoch": 15.8125, + "grad_norm": 0.00046108156675472856, + "learning_rate": 5.9641160166592026e-05, + "loss": 8.884572707756888e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 1012, + "tokens/total": 15279436, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 327178 + }, + { + "epoch": 15.828125, + "grad_norm": 0.00011094512592535466, + "learning_rate": 5.9568894253544874e-05, + "loss": 3.8291545934043825e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1013, + "tokens/total": 15294388, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 327502 + }, + { + "epoch": 15.84375, + "grad_norm": 0.00011913081834791228, + "learning_rate": 5.9496616432874466e-05, + "loss": 3.260595121901133e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1014, + "tokens/total": 15309404, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 327821 + }, + { + "epoch": 15.859375, + "grad_norm": 8.121941209537908e-05, + "learning_rate": 5.9424326892953975e-05, + "loss": 2.8619911063287873e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1015, + "tokens/total": 15324568, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 328143 + }, + { + "epoch": 15.875, + "grad_norm": 8.115295349853113e-05, + "learning_rate": 5.935202582218707e-05, + "loss": 3.0984724617155734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1016, + "tokens/total": 15339560, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 328462 + }, + { + "epoch": 15.890625, + "grad_norm": 9.505224443273619e-05, + "learning_rate": 5.9279713409007555e-05, + "loss": 3.3673372854536865e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1017, + "tokens/total": 15354696, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 328785 + }, + { + "epoch": 15.90625, + "grad_norm": 0.00014480789832305163, + "learning_rate": 5.92073898418787e-05, + "loss": 4.5865167521697e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1018, + "tokens/total": 15369816, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 329108 + }, + { + "epoch": 15.921875, + "grad_norm": 0.00011009223817382008, + "learning_rate": 5.9135055309292955e-05, + "loss": 3.6172034469927894e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1019, + "tokens/total": 15384936, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 329429 + }, + { + "epoch": 15.9375, + "grad_norm": 0.00018365232972428203, + "learning_rate": 5.906270999977121e-05, + "loss": 4.949197318637744e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1020, + "tokens/total": 15400068, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 329751 + }, + { + "epoch": 15.953125, + "grad_norm": 9.444593160878867e-05, + "learning_rate": 5.899035410186259e-05, + "loss": 2.979335931740934e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1021, + "tokens/total": 15415120, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 330075 + }, + { + "epoch": 15.96875, + "grad_norm": 7.64124997658655e-05, + "learning_rate": 5.891798780414369e-05, + "loss": 3.143222329526907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1022, + "tokens/total": 15430256, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 330394 + }, + { + "epoch": 15.984375, + "grad_norm": 6.94402915542014e-05, + "learning_rate": 5.884561129521832e-05, + "loss": 2.8725444281008095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1023, + "tokens/total": 15445400, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 330717 + }, + { + "epoch": 16.0, + "grad_norm": 0.00016600823437329382, + "learning_rate": 5.8773224763716795e-05, + "loss": 3.3972441997320857e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1024, + "tokens/total": 15460436, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 331040 + }, + { + "epoch": 16.015625, + "grad_norm": 0.00013672508066520095, + "learning_rate": 5.8700828398295636e-05, + "loss": 3.5022358133574016e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1025, + "tokens/total": 15475588, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 331364 + }, + { + "epoch": 16.03125, + "grad_norm": 7.496897887904197e-05, + "learning_rate": 5.862842238763694e-05, + "loss": 2.8085612484574085e-06, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1026, + "tokens/total": 15490656, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 331683 + }, + { + "epoch": 16.046875, + "grad_norm": 0.00018603302305564284, + "learning_rate": 5.855600692044797e-05, + "loss": 4.103082574147265e-06, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1027, + "tokens/total": 15505588, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 332008 + }, + { + "epoch": 16.0625, + "grad_norm": 6.286927236942574e-05, + "learning_rate": 5.8483582185460636e-05, + "loss": 2.4775984002189944e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1028, + "tokens/total": 15520748, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 332331 + }, + { + "epoch": 16.078125, + "grad_norm": 6.979356112424284e-05, + "learning_rate": 5.8411148371431e-05, + "loss": 2.7643779958452797e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1029, + "tokens/total": 15535884, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 332654 + }, + { + "epoch": 16.09375, + "grad_norm": 7.733966049272567e-05, + "learning_rate": 5.833870566713876e-05, + "loss": 3.0545829758921172e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1030, + "tokens/total": 15551104, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 332976 + }, + { + "epoch": 16.109375, + "grad_norm": 0.0001807802909752354, + "learning_rate": 5.826625426138681e-05, + "loss": 4.9547088565304875e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1031, + "tokens/total": 15566260, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 333299 + }, + { + "epoch": 16.125, + "grad_norm": 8.790877473074943e-05, + "learning_rate": 5.8193794343000715e-05, + "loss": 3.052951115023461e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1032, + "tokens/total": 15581488, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 333624 + }, + { + "epoch": 16.140625, + "grad_norm": 6.65875559207052e-05, + "learning_rate": 5.812132610082825e-05, + "loss": 2.5883960006467532e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1033, + "tokens/total": 15596436, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 333946 + }, + { + "epoch": 16.15625, + "grad_norm": 0.00015213366714306176, + "learning_rate": 5.804884972373882e-05, + "loss": 3.1400043098983588e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1034, + "tokens/total": 15611532, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 334268 + }, + { + "epoch": 16.171875, + "grad_norm": 9.213191515300423e-05, + "learning_rate": 5.79763654006231e-05, + "loss": 3.1738866255182074e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1035, + "tokens/total": 15626552, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 334588 + }, + { + "epoch": 16.1875, + "grad_norm": 8.001571404747665e-05, + "learning_rate": 5.790387332039244e-05, + "loss": 2.317058033440844e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1036, + "tokens/total": 15641640, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 334911 + }, + { + "epoch": 16.203125, + "grad_norm": 0.00010642761480994523, + "learning_rate": 5.783137367197842e-05, + "loss": 2.5900023956637597e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1037, + "tokens/total": 15656820, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 335233 + }, + { + "epoch": 16.21875, + "grad_norm": 5.980083369649947e-05, + "learning_rate": 5.775886664433232e-05, + "loss": 2.3242796487465966e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1038, + "tokens/total": 15671984, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 335555 + }, + { + "epoch": 16.234375, + "grad_norm": 0.00019549434364307672, + "learning_rate": 5.7686352426424696e-05, + "loss": 3.7035438253951725e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1039, + "tokens/total": 15687116, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 335882 + }, + { + "epoch": 16.25, + "grad_norm": 5.668194717145525e-05, + "learning_rate": 5.76138312072448e-05, + "loss": 2.2780259314458817e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1040, + "tokens/total": 15702184, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 336205 + }, + { + "epoch": 16.265625, + "grad_norm": 6.422160367947072e-05, + "learning_rate": 5.7541303175800156e-05, + "loss": 2.2468607312475797e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1041, + "tokens/total": 15717188, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 336528 + }, + { + "epoch": 16.28125, + "grad_norm": 6.662744999630377e-05, + "learning_rate": 5.746876852111602e-05, + "loss": 2.612257048895117e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1042, + "tokens/total": 15732344, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 336855 + }, + { + "epoch": 16.296875, + "grad_norm": 5.637756839860231e-05, + "learning_rate": 5.739622743223494e-05, + "loss": 2.418863005004823e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1043, + "tokens/total": 15747420, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 337177 + }, + { + "epoch": 16.3125, + "grad_norm": 5.679674359271303e-05, + "learning_rate": 5.7323680098216215e-05, + "loss": 2.1683340492018033e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1044, + "tokens/total": 15762480, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 337498 + }, + { + "epoch": 16.328125, + "grad_norm": 5.147862611920573e-05, + "learning_rate": 5.725112670813543e-05, + "loss": 2.1631653908116277e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1045, + "tokens/total": 15777524, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 337819 + }, + { + "epoch": 16.34375, + "grad_norm": 5.788360795122571e-05, + "learning_rate": 5.717856745108393e-05, + "loss": 2.3735460672469344e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1046, + "tokens/total": 15792672, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 338138 + }, + { + "epoch": 16.359375, + "grad_norm": 7.761285814922303e-05, + "learning_rate": 5.710600251616838e-05, + "loss": 2.6638558665581513e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1047, + "tokens/total": 15807740, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 338462 + }, + { + "epoch": 16.375, + "grad_norm": 5.9256795793771744e-05, + "learning_rate": 5.703343209251023e-05, + "loss": 2.274974349347758e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1048, + "tokens/total": 15822804, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 338787 + }, + { + "epoch": 16.390625, + "grad_norm": 6.629613926634192e-05, + "learning_rate": 5.696085636924522e-05, + "loss": 2.316260179213714e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1049, + "tokens/total": 15837876, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 339115 + }, + { + "epoch": 16.40625, + "grad_norm": 8.543812145944685e-05, + "learning_rate": 5.688827553552294e-05, + "loss": 3.229639105484239e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1050, + "tokens/total": 15853020, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 339438 + }, + { + "epoch": 16.421875, + "grad_norm": 6.60706136841327e-05, + "learning_rate": 5.681568978050624e-05, + "loss": 2.4331479835382197e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1051, + "tokens/total": 15868256, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 339761 + }, + { + "epoch": 16.4375, + "grad_norm": 0.00011383193486835808, + "learning_rate": 5.6743099293370884e-05, + "loss": 2.7795824735221686e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1052, + "tokens/total": 15883452, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 340082 + }, + { + "epoch": 16.453125, + "grad_norm": 0.00010293241939507425, + "learning_rate": 5.667050426330488e-05, + "loss": 3.657840807136381e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1053, + "tokens/total": 15898444, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 340401 + }, + { + "epoch": 16.46875, + "grad_norm": 6.830687925685197e-05, + "learning_rate": 5.659790487950812e-05, + "loss": 2.5762624318304006e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1054, + "tokens/total": 15913516, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 340725 + }, + { + "epoch": 16.484375, + "grad_norm": 7.307613850571215e-05, + "learning_rate": 5.652530133119183e-05, + "loss": 1.8700184227782302e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1055, + "tokens/total": 15928648, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 341046 + }, + { + "epoch": 16.5, + "grad_norm": 5.11426042066887e-05, + "learning_rate": 5.645269380757813e-05, + "loss": 2.010842763411347e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1056, + "tokens/total": 15943728, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 341370 + }, + { + "epoch": 16.515625, + "grad_norm": 8.026239811442792e-05, + "learning_rate": 5.6380082497899414e-05, + "loss": 2.773282631096663e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1057, + "tokens/total": 15958892, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 341695 + }, + { + "epoch": 16.53125, + "grad_norm": 0.00010482090146979317, + "learning_rate": 5.630746759139803e-05, + "loss": 2.8316499083302915e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1058, + "tokens/total": 15974052, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 342017 + }, + { + "epoch": 16.546875, + "grad_norm": 7.734580140095204e-05, + "learning_rate": 5.6234849277325675e-05, + "loss": 2.8357806058920687e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1059, + "tokens/total": 15989292, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 342346 + }, + { + "epoch": 16.5625, + "grad_norm": 5.8893219829769805e-05, + "learning_rate": 5.616222774494289e-05, + "loss": 2.450640067763743e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1060, + "tokens/total": 16004404, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 342672 + }, + { + "epoch": 16.578125, + "grad_norm": 7.280950376298279e-05, + "learning_rate": 5.608960318351863e-05, + "loss": 2.652550392667763e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1061, + "tokens/total": 16019580, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 342996 + }, + { + "epoch": 16.59375, + "grad_norm": 6.881778972456232e-05, + "learning_rate": 5.601697578232978e-05, + "loss": 2.1431069399113767e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1062, + "tokens/total": 16034716, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 343322 + }, + { + "epoch": 16.609375, + "grad_norm": 0.0001707331684883684, + "learning_rate": 5.594434573066056e-05, + "loss": 3.876075879816199e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1063, + "tokens/total": 16049772, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 343645 + }, + { + "epoch": 16.625, + "grad_norm": 5.8329813327873126e-05, + "learning_rate": 5.587171321780216e-05, + "loss": 2.1158343770366628e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1064, + "tokens/total": 16064916, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 343972 + }, + { + "epoch": 16.640625, + "grad_norm": 5.108560435473919e-05, + "learning_rate": 5.579907843305212e-05, + "loss": 2.1045295852673007e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1065, + "tokens/total": 16080152, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 344299 + }, + { + "epoch": 16.65625, + "grad_norm": 7.851166446926072e-05, + "learning_rate": 5.572644156571396e-05, + "loss": 3.0778667223785305e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1066, + "tokens/total": 16095196, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 344624 + }, + { + "epoch": 16.671875, + "grad_norm": 5.4422638640971854e-05, + "learning_rate": 5.565380280509659e-05, + "loss": 1.912759216793347e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1067, + "tokens/total": 16110188, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 344948 + }, + { + "epoch": 16.6875, + "grad_norm": 0.00020886174752376974, + "learning_rate": 5.558116234051387e-05, + "loss": 4.328313934820471e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1068, + "tokens/total": 16125300, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 345270 + }, + { + "epoch": 16.703125, + "grad_norm": 6.223680247785524e-05, + "learning_rate": 5.55085203612841e-05, + "loss": 2.4240298444055952e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1069, + "tokens/total": 16140324, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 345595 + }, + { + "epoch": 16.71875, + "grad_norm": 4.3806780013255775e-05, + "learning_rate": 5.543587705672952e-05, + "loss": 1.7543569583722274e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1070, + "tokens/total": 16155484, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 345921 + }, + { + "epoch": 16.734375, + "grad_norm": 6.738673255313188e-05, + "learning_rate": 5.5363232616175845e-05, + "loss": 2.6232712571072625e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1071, + "tokens/total": 16170692, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 346250 + }, + { + "epoch": 16.75, + "grad_norm": 7.497112528653815e-05, + "learning_rate": 5.529058722895172e-05, + "loss": 2.5162707970594056e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1072, + "tokens/total": 16185756, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 346575 + }, + { + "epoch": 16.765625, + "grad_norm": 6.020827277097851e-05, + "learning_rate": 5.521794108438828e-05, + "loss": 2.397573780399398e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1073, + "tokens/total": 16200856, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 346897 + }, + { + "epoch": 16.78125, + "grad_norm": 5.4258944146567956e-05, + "learning_rate": 5.5145294371818637e-05, + "loss": 2.0468289676500717e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1074, + "tokens/total": 16215896, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 347221 + }, + { + "epoch": 16.796875, + "grad_norm": 8.13579754321836e-05, + "learning_rate": 5.507264728057734e-05, + "loss": 2.446963890179177e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1075, + "tokens/total": 16231020, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 347544 + }, + { + "epoch": 16.8125, + "grad_norm": 0.00011984182492597029, + "learning_rate": 5.500000000000001e-05, + "loss": 2.3698257791693322e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1076, + "tokens/total": 16246048, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 347865 + }, + { + "epoch": 16.828125, + "grad_norm": 7.277093391167e-05, + "learning_rate": 5.492735271942265e-05, + "loss": 2.239054310848587e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1077, + "tokens/total": 16261216, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 348192 + }, + { + "epoch": 16.84375, + "grad_norm": 5.739858534070663e-05, + "learning_rate": 5.485470562818139e-05, + "loss": 2.2521435312228277e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1078, + "tokens/total": 16276292, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 348512 + }, + { + "epoch": 16.859375, + "grad_norm": 5.590792716247961e-05, + "learning_rate": 5.4782058915611733e-05, + "loss": 2.234276280432823e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1079, + "tokens/total": 16291260, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 348829 + }, + { + "epoch": 16.875, + "grad_norm": 0.0003715770726557821, + "learning_rate": 5.4709412771048286e-05, + "loss": 7.504874247388216e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 1080, + "tokens/total": 16306500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 349155 + }, + { + "epoch": 16.890625, + "grad_norm": 5.7440734963165596e-05, + "learning_rate": 5.463676738382416e-05, + "loss": 2.2509959762828657e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1081, + "tokens/total": 16321600, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 349477 + }, + { + "epoch": 16.90625, + "grad_norm": 0.0002011326141655445, + "learning_rate": 5.45641229432705e-05, + "loss": 3.221858605684247e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1082, + "tokens/total": 16336680, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 349798 + }, + { + "epoch": 16.921875, + "grad_norm": 4.970157897332683e-05, + "learning_rate": 5.449147963871593e-05, + "loss": 1.8949483546748525e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1083, + "tokens/total": 16351764, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 350121 + }, + { + "epoch": 16.9375, + "grad_norm": 7.564233237644657e-05, + "learning_rate": 5.441883765948615e-05, + "loss": 2.593346152934828e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1084, + "tokens/total": 16366760, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 350441 + }, + { + "epoch": 16.953125, + "grad_norm": 9.053905523614958e-05, + "learning_rate": 5.434619719490342e-05, + "loss": 2.8712870516756084e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1085, + "tokens/total": 16381844, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 350763 + }, + { + "epoch": 16.96875, + "grad_norm": 0.00017584952001925558, + "learning_rate": 5.4273558434286064e-05, + "loss": 3.97955682274187e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1086, + "tokens/total": 16396936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 351083 + }, + { + "epoch": 16.984375, + "grad_norm": 6.725845742039382e-05, + "learning_rate": 5.420092156694789e-05, + "loss": 2.5975857624871423e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1087, + "tokens/total": 16411896, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 351404 + }, + { + "epoch": 17.0, + "grad_norm": 0.00011377281771274284, + "learning_rate": 5.412828678219786e-05, + "loss": 3.4532884001237107e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1088, + "tokens/total": 16427004, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 351730 + }, + { + "epoch": 17.015625, + "grad_norm": 6.486582424258813e-05, + "learning_rate": 5.405565426933944e-05, + "loss": 2.474385382811306e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1089, + "tokens/total": 16442060, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 352055 + }, + { + "epoch": 17.03125, + "grad_norm": 0.00013874955766368657, + "learning_rate": 5.398302421767024e-05, + "loss": 3.339830982440617e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1090, + "tokens/total": 16457100, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 352372 + }, + { + "epoch": 17.046875, + "grad_norm": 0.00018217696924693882, + "learning_rate": 5.391039681648138e-05, + "loss": 4.669335794460494e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1091, + "tokens/total": 16472180, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 352692 + }, + { + "epoch": 17.0625, + "grad_norm": 4.704017555923201e-05, + "learning_rate": 5.383777225505713e-05, + "loss": 1.6187043456739048e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1092, + "tokens/total": 16487360, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 353021 + }, + { + "epoch": 17.078125, + "grad_norm": 0.00016949341807048768, + "learning_rate": 5.376515072267434e-05, + "loss": 2.3408824745274615e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1093, + "tokens/total": 16502416, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 353344 + }, + { + "epoch": 17.09375, + "grad_norm": 5.314912777976133e-05, + "learning_rate": 5.369253240860198e-05, + "loss": 2.1153775833226973e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1094, + "tokens/total": 16517468, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 353665 + }, + { + "epoch": 17.109375, + "grad_norm": 3.976970401708968e-05, + "learning_rate": 5.3619917502100604e-05, + "loss": 1.5823655985514051e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1095, + "tokens/total": 16532616, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 353989 + }, + { + "epoch": 17.125, + "grad_norm": 5.37064261152409e-05, + "learning_rate": 5.3547306192421895e-05, + "loss": 2.0994102669646963e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1096, + "tokens/total": 16547648, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 354315 + }, + { + "epoch": 17.140625, + "grad_norm": 5.0919614295708016e-05, + "learning_rate": 5.3474698668808175e-05, + "loss": 1.8454038581694476e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1097, + "tokens/total": 16562856, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 354637 + }, + { + "epoch": 17.15625, + "grad_norm": 7.009955879766494e-05, + "learning_rate": 5.34020951204919e-05, + "loss": 2.1261653273541015e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1098, + "tokens/total": 16577892, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 354957 + }, + { + "epoch": 17.171875, + "grad_norm": 6.461940211011097e-05, + "learning_rate": 5.332949573669515e-05, + "loss": 2.0826241780014243e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1099, + "tokens/total": 16592988, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 355279 + }, + { + "epoch": 17.1875, + "grad_norm": 4.3906700739171356e-05, + "learning_rate": 5.3256900706629133e-05, + "loss": 1.7440319197703502e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1100, + "tokens/total": 16608128, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 355603 + }, + { + "epoch": 17.203125, + "grad_norm": 0.00011730727419489995, + "learning_rate": 5.318431021949376e-05, + "loss": 3.0324697490868857e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1101, + "tokens/total": 16623244, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 355925 + }, + { + "epoch": 17.21875, + "grad_norm": 0.0004744554462376982, + "learning_rate": 5.3111724464477084e-05, + "loss": 5.75247122469591e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 1102, + "tokens/total": 16638224, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 356247 + }, + { + "epoch": 17.234375, + "grad_norm": 7.133598410291597e-05, + "learning_rate": 5.3039143630754796e-05, + "loss": 2.567876890680054e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1103, + "tokens/total": 16653332, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 356571 + }, + { + "epoch": 17.25, + "grad_norm": 7.450963312294334e-05, + "learning_rate": 5.2966567907489794e-05, + "loss": 2.3665807020734064e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1104, + "tokens/total": 16668436, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 356893 + }, + { + "epoch": 17.265625, + "grad_norm": 5.1341005018912256e-05, + "learning_rate": 5.2893997483831634e-05, + "loss": 1.8750467916106572e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1105, + "tokens/total": 16683560, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 357216 + }, + { + "epoch": 17.28125, + "grad_norm": 0.00013866904191672802, + "learning_rate": 5.282143254891609e-05, + "loss": 3.3119247291324427e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1106, + "tokens/total": 16698772, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 357538 + }, + { + "epoch": 17.296875, + "grad_norm": 0.00010163118713535368, + "learning_rate": 5.274887329186459e-05, + "loss": 2.937394810942351e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1107, + "tokens/total": 16713868, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 357862 + }, + { + "epoch": 17.3125, + "grad_norm": 0.00011721099144779146, + "learning_rate": 5.267631990178379e-05, + "loss": 3.690820904012071e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1108, + "tokens/total": 16729000, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 358187 + }, + { + "epoch": 17.328125, + "grad_norm": 0.00014858219947200269, + "learning_rate": 5.260377256776506e-05, + "loss": 3.1422362098965095e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1109, + "tokens/total": 16743964, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 358501 + }, + { + "epoch": 17.34375, + "grad_norm": 5.4628912039333954e-05, + "learning_rate": 5.2531231478884e-05, + "loss": 1.9555757262423867e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1110, + "tokens/total": 16759096, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 358827 + }, + { + "epoch": 17.359375, + "grad_norm": 4.646389788831584e-05, + "learning_rate": 5.245869682419986e-05, + "loss": 1.7461304651078535e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1111, + "tokens/total": 16774104, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 359154 + }, + { + "epoch": 17.375, + "grad_norm": 6.257047789404169e-05, + "learning_rate": 5.238616879275521e-05, + "loss": 2.5145702693407657e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1112, + "tokens/total": 16789204, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 359476 + }, + { + "epoch": 17.390625, + "grad_norm": 6.670400034636259e-05, + "learning_rate": 5.231364757357532e-05, + "loss": 2.1189086965023307e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1113, + "tokens/total": 16804280, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 359799 + }, + { + "epoch": 17.40625, + "grad_norm": 8.855111809680238e-05, + "learning_rate": 5.224113335566769e-05, + "loss": 2.779685246423469e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1114, + "tokens/total": 16819244, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 360121 + }, + { + "epoch": 17.421875, + "grad_norm": 5.411976235336624e-05, + "learning_rate": 5.21686263280216e-05, + "loss": 1.975502982531907e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1115, + "tokens/total": 16834324, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 360442 + }, + { + "epoch": 17.4375, + "grad_norm": 4.62812531623058e-05, + "learning_rate": 5.209612667960757e-05, + "loss": 1.836690898926463e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1116, + "tokens/total": 16849412, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 360765 + }, + { + "epoch": 17.453125, + "grad_norm": 3.681908856378868e-05, + "learning_rate": 5.2023634599376926e-05, + "loss": 1.4974585837990162e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1117, + "tokens/total": 16864408, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 361097 + }, + { + "epoch": 17.46875, + "grad_norm": 8.148687629727647e-05, + "learning_rate": 5.195115027626119e-05, + "loss": 2.8984181881241966e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1118, + "tokens/total": 16879600, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 361419 + }, + { + "epoch": 17.484375, + "grad_norm": 6.114594725659117e-05, + "learning_rate": 5.187867389917177e-05, + "loss": 2.0753905118908733e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1119, + "tokens/total": 16894664, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 361744 + }, + { + "epoch": 17.5, + "grad_norm": 0.00014562737487722188, + "learning_rate": 5.1806205656999295e-05, + "loss": 3.264980705353082e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1120, + "tokens/total": 16909820, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 362068 + }, + { + "epoch": 17.515625, + "grad_norm": 7.160433597164229e-05, + "learning_rate": 5.173374573861322e-05, + "loss": 2.160111080229399e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1121, + "tokens/total": 16924896, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 362389 + }, + { + "epoch": 17.53125, + "grad_norm": 3.812456634477712e-05, + "learning_rate": 5.166129433286125e-05, + "loss": 1.4740290907866438e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.02, + "memory/max_allocated (GiB)": 33.02, + "ppl": 1.0, + "step": 1122, + "tokens/total": 16940088, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 362709 + }, + { + "epoch": 17.546875, + "grad_norm": 4.7998109948821366e-05, + "learning_rate": 5.1588851628569024e-05, + "loss": 1.8353025552642066e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1123, + "tokens/total": 16955248, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 363033 + }, + { + "epoch": 17.5625, + "grad_norm": 5.361408329918049e-05, + "learning_rate": 5.151641781453936e-05, + "loss": 2.049611111942795e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1124, + "tokens/total": 16970420, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 363357 + }, + { + "epoch": 17.578125, + "grad_norm": 4.882154462393373e-05, + "learning_rate": 5.1443993079552045e-05, + "loss": 1.7708264294924447e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1125, + "tokens/total": 16985544, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 363681 + }, + { + "epoch": 17.59375, + "grad_norm": 5.3644802392227575e-05, + "learning_rate": 5.137157761236308e-05, + "loss": 2.1348716927604983e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1126, + "tokens/total": 17000660, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 364005 + }, + { + "epoch": 17.609375, + "grad_norm": 6.806557212257758e-05, + "learning_rate": 5.129917160170439e-05, + "loss": 1.7366216980008176e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1127, + "tokens/total": 17015752, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 364331 + }, + { + "epoch": 17.625, + "grad_norm": 0.00010055719030788168, + "learning_rate": 5.122677523628321e-05, + "loss": 2.225175194325857e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1128, + "tokens/total": 17030796, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 364655 + }, + { + "epoch": 17.640625, + "grad_norm": 4.990952220396139e-05, + "learning_rate": 5.1154388704781705e-05, + "loss": 1.8721705146163004e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1129, + "tokens/total": 17045880, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 364980 + }, + { + "epoch": 17.65625, + "grad_norm": 7.897414616309106e-05, + "learning_rate": 5.108201219585631e-05, + "loss": 2.4452153866150184e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1130, + "tokens/total": 17061068, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 365303 + }, + { + "epoch": 17.671875, + "grad_norm": 5.068585596745834e-05, + "learning_rate": 5.100964589813742e-05, + "loss": 1.803147256396187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1131, + "tokens/total": 17076200, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 365624 + }, + { + "epoch": 17.6875, + "grad_norm": 4.330592855694704e-05, + "learning_rate": 5.093729000022879e-05, + "loss": 1.576580530127103e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1132, + "tokens/total": 17091204, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 365948 + }, + { + "epoch": 17.703125, + "grad_norm": 4.43206045019906e-05, + "learning_rate": 5.0864944690707076e-05, + "loss": 1.7179522728838492e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1133, + "tokens/total": 17106228, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 366273 + }, + { + "epoch": 17.71875, + "grad_norm": 6.410075002349913e-05, + "learning_rate": 5.079261015812131e-05, + "loss": 2.2823087419965304e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1134, + "tokens/total": 17121480, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 366591 + }, + { + "epoch": 17.734375, + "grad_norm": 6.095452044974081e-05, + "learning_rate": 5.072028659099246e-05, + "loss": 2.0786696950381156e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1135, + "tokens/total": 17136436, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 366912 + }, + { + "epoch": 17.75, + "grad_norm": 0.0001288848288822919, + "learning_rate": 5.0647974177812926e-05, + "loss": 2.757469701464288e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1136, + "tokens/total": 17151492, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 367238 + }, + { + "epoch": 17.765625, + "grad_norm": 5.42581983609125e-05, + "learning_rate": 5.057567310704605e-05, + "loss": 1.964313014468644e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1137, + "tokens/total": 17166708, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 367560 + }, + { + "epoch": 17.78125, + "grad_norm": 7.874661969253793e-05, + "learning_rate": 5.050338356712555e-05, + "loss": 1.934493411681615e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1138, + "tokens/total": 17181856, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 367887 + }, + { + "epoch": 17.796875, + "grad_norm": 5.227981091593392e-05, + "learning_rate": 5.043110574645514e-05, + "loss": 2.1090099835419096e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1139, + "tokens/total": 17196996, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 368209 + }, + { + "epoch": 17.8125, + "grad_norm": 4.149093001615256e-05, + "learning_rate": 5.0358839833408e-05, + "loss": 1.5239410231515649e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1140, + "tokens/total": 17212124, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 368535 + }, + { + "epoch": 17.828125, + "grad_norm": 0.00014195829862728715, + "learning_rate": 5.0286586016326244e-05, + "loss": 2.5105987333517987e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1141, + "tokens/total": 17227176, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 368854 + }, + { + "epoch": 17.84375, + "grad_norm": 0.00012074544065399095, + "learning_rate": 5.021434448352047e-05, + "loss": 1.3399114777712384e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1142, + "tokens/total": 17242328, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 369179 + }, + { + "epoch": 17.859375, + "grad_norm": 4.588439333019778e-05, + "learning_rate": 5.014211542326928e-05, + "loss": 1.6399665128119523e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1143, + "tokens/total": 17257444, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 369503 + }, + { + "epoch": 17.875, + "grad_norm": 4.1756204154808074e-05, + "learning_rate": 5.006989902381873e-05, + "loss": 1.579968852638558e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1144, + "tokens/total": 17272628, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 369826 + }, + { + "epoch": 17.890625, + "grad_norm": 0.00010401308099972084, + "learning_rate": 4.999769547338196e-05, + "loss": 1.5226391951728147e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1145, + "tokens/total": 17287644, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 370157 + }, + { + "epoch": 17.90625, + "grad_norm": 8.81893647601828e-05, + "learning_rate": 4.99255049601385e-05, + "loss": 2.3466259335691575e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1146, + "tokens/total": 17302800, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 370481 + }, + { + "epoch": 17.921875, + "grad_norm": 3.904552067979239e-05, + "learning_rate": 4.9853327672234e-05, + "loss": 1.3858089005225338e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1147, + "tokens/total": 17317872, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 370806 + }, + { + "epoch": 17.9375, + "grad_norm": 5.618481736746617e-05, + "learning_rate": 4.97811637977796e-05, + "loss": 1.997294248212711e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1148, + "tokens/total": 17332960, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 371131 + }, + { + "epoch": 17.953125, + "grad_norm": 4.3289295717841014e-05, + "learning_rate": 4.970901352485152e-05, + "loss": 1.5739980199214187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1149, + "tokens/total": 17348028, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 371455 + }, + { + "epoch": 17.96875, + "grad_norm": 4.6656281483592466e-05, + "learning_rate": 4.963687704149047e-05, + "loss": 1.8726941561908461e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1150, + "tokens/total": 17363192, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 371777 + }, + { + "epoch": 17.984375, + "grad_norm": 6.316841609077528e-05, + "learning_rate": 4.9564754535701255e-05, + "loss": 2.057156052615028e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1151, + "tokens/total": 17378380, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 372097 + }, + { + "epoch": 18.0, + "grad_norm": 9.153840801445767e-05, + "learning_rate": 4.9492646195452284e-05, + "loss": 2.148427938664099e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1152, + "tokens/total": 17393556, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 372420 + }, + { + "epoch": 18.015625, + "grad_norm": 5.5113298003561795e-05, + "learning_rate": 4.9420552208674964e-05, + "loss": 1.8422852008370683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1153, + "tokens/total": 17408624, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 372740 + }, + { + "epoch": 18.03125, + "grad_norm": 6.566980300704017e-05, + "learning_rate": 4.934847276326337e-05, + "loss": 1.5596227740388713e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1154, + "tokens/total": 17423700, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 373062 + }, + { + "epoch": 18.046875, + "grad_norm": 3.41080994985532e-05, + "learning_rate": 4.927640804707364e-05, + "loss": 1.3603357729152776e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1155, + "tokens/total": 17438852, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 373385 + }, + { + "epoch": 18.0625, + "grad_norm": 4.892497599939816e-05, + "learning_rate": 4.9204358247923573e-05, + "loss": 1.894281240311102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1156, + "tokens/total": 17453868, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 373709 + }, + { + "epoch": 18.078125, + "grad_norm": 0.0001309289364144206, + "learning_rate": 4.913232355359201e-05, + "loss": 2.644880169100361e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1157, + "tokens/total": 17468880, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 374029 + }, + { + "epoch": 18.09375, + "grad_norm": 5.326896280166693e-05, + "learning_rate": 4.906030415181847e-05, + "loss": 1.6114896652652533e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1158, + "tokens/total": 17483868, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 374353 + }, + { + "epoch": 18.109375, + "grad_norm": 0.00013752431550528854, + "learning_rate": 4.8988300230302634e-05, + "loss": 2.203453277616063e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1159, + "tokens/total": 17499032, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 374679 + }, + { + "epoch": 18.125, + "grad_norm": 4.509186328505166e-05, + "learning_rate": 4.891631197670381e-05, + "loss": 1.5230970120683196e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1160, + "tokens/total": 17514008, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 375000 + }, + { + "epoch": 18.140625, + "grad_norm": 6.77557909511961e-05, + "learning_rate": 4.8844339578640484e-05, + "loss": 1.9931019323848886e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1161, + "tokens/total": 17529212, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 375325 + }, + { + "epoch": 18.15625, + "grad_norm": 0.0001433398574590683, + "learning_rate": 4.87723832236898e-05, + "loss": 2.562797590144328e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1162, + "tokens/total": 17544304, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 375650 + }, + { + "epoch": 18.171875, + "grad_norm": 3.762907363125123e-05, + "learning_rate": 4.8700443099387115e-05, + "loss": 1.4893737443344435e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1163, + "tokens/total": 17559500, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 375974 + }, + { + "epoch": 18.1875, + "grad_norm": 0.00012924327165819705, + "learning_rate": 4.8628519393225494e-05, + "loss": 1.8429645933792926e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1164, + "tokens/total": 17574504, + "tokens/train_per_sec_per_gpu": 8.96, + "tokens/trainable": 376294 + }, + { + "epoch": 18.203125, + "grad_norm": 8.961712592281401e-05, + "learning_rate": 4.855661229265516e-05, + "loss": 2.4633820885355817e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1165, + "tokens/total": 17589612, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 376614 + }, + { + "epoch": 18.21875, + "grad_norm": 5.846893327543512e-05, + "learning_rate": 4.848472198508308e-05, + "loss": 1.7539060763738235e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1166, + "tokens/total": 17604728, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 376937 + }, + { + "epoch": 18.234375, + "grad_norm": 9.194444282911718e-05, + "learning_rate": 4.84128486578725e-05, + "loss": 2.48921537604474e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1167, + "tokens/total": 17619828, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 377262 + }, + { + "epoch": 18.25, + "grad_norm": 7.207986345747486e-05, + "learning_rate": 4.834099249834238e-05, + "loss": 2.2066833480494097e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1168, + "tokens/total": 17634884, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 377581 + }, + { + "epoch": 18.265625, + "grad_norm": 5.99343293288257e-05, + "learning_rate": 4.826915369376691e-05, + "loss": 1.5838352283026325e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1169, + "tokens/total": 17650028, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 377906 + }, + { + "epoch": 18.28125, + "grad_norm": 2.9692891985177994e-05, + "learning_rate": 4.819733243137508e-05, + "loss": 1.1636724366326234e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1170, + "tokens/total": 17665088, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 378228 + }, + { + "epoch": 18.296875, + "grad_norm": 4.723759775515646e-05, + "learning_rate": 4.8125528898350135e-05, + "loss": 1.755090579536045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1171, + "tokens/total": 17680180, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 378551 + }, + { + "epoch": 18.3125, + "grad_norm": 5.634744229610078e-05, + "learning_rate": 4.805374328182918e-05, + "loss": 1.9587191673053894e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1172, + "tokens/total": 17695212, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 378879 + }, + { + "epoch": 18.328125, + "grad_norm": 4.2808173020603135e-05, + "learning_rate": 4.798197576890252e-05, + "loss": 1.456910922570387e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1173, + "tokens/total": 17710364, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 379205 + }, + { + "epoch": 18.34375, + "grad_norm": 0.0002998764393851161, + "learning_rate": 4.791022654661336e-05, + "loss": 5.216873432800639e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.00001, + "step": 1174, + "tokens/total": 17725380, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 379530 + }, + { + "epoch": 18.359375, + "grad_norm": 0.00010181423567701131, + "learning_rate": 4.7838495801957214e-05, + "loss": 2.466581690896419e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1175, + "tokens/total": 17740320, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 379855 + }, + { + "epoch": 18.375, + "grad_norm": 3.664367613964714e-05, + "learning_rate": 4.7766783721881436e-05, + "loss": 1.4246884347812738e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1176, + "tokens/total": 17755380, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 380186 + }, + { + "epoch": 18.390625, + "grad_norm": 4.097115379408933e-05, + "learning_rate": 4.7695090493284724e-05, + "loss": 1.5918500366751687e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1177, + "tokens/total": 17770508, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 380511 + }, + { + "epoch": 18.40625, + "grad_norm": 0.00010890235716942698, + "learning_rate": 4.762341630301666e-05, + "loss": 1.3999627981320373e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1178, + "tokens/total": 17785644, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 380831 + }, + { + "epoch": 18.421875, + "grad_norm": 4.751495362143032e-05, + "learning_rate": 4.7551761337877175e-05, + "loss": 1.826504671953444e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1179, + "tokens/total": 17800712, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 381153 + }, + { + "epoch": 18.4375, + "grad_norm": 4.724668906419538e-05, + "learning_rate": 4.748012578461618e-05, + "loss": 1.6813839920359896e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1180, + "tokens/total": 17815812, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 381475 + }, + { + "epoch": 18.453125, + "grad_norm": 4.947883280692622e-05, + "learning_rate": 4.740850982993288e-05, + "loss": 1.6814337868709117e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1181, + "tokens/total": 17830848, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 381798 + }, + { + "epoch": 18.46875, + "grad_norm": 8.495483780279756e-05, + "learning_rate": 4.733691366047548e-05, + "loss": 1.8123433846994885e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1182, + "tokens/total": 17845968, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 382119 + }, + { + "epoch": 18.484375, + "grad_norm": 0.0001198833851958625, + "learning_rate": 4.72653374628406e-05, + "loss": 2.74478452411131e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1183, + "tokens/total": 17861116, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 382443 + }, + { + "epoch": 18.5, + "grad_norm": 5.8095407439395785e-05, + "learning_rate": 4.7193781423572815e-05, + "loss": 1.343330609415716e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1184, + "tokens/total": 17876312, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 382767 + }, + { + "epoch": 18.515625, + "grad_norm": 6.00434941588901e-05, + "learning_rate": 4.7122245729164115e-05, + "loss": 1.8043461977867992e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1185, + "tokens/total": 17891544, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 383090 + }, + { + "epoch": 18.53125, + "grad_norm": 7.066176476655528e-05, + "learning_rate": 4.705073056605355e-05, + "loss": 1.895393666018208e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1186, + "tokens/total": 17906680, + "tokens/train_per_sec_per_gpu": 10.29, + "tokens/trainable": 383418 + }, + { + "epoch": 18.546875, + "grad_norm": 3.315817957627587e-05, + "learning_rate": 4.697923612062659e-05, + "loss": 1.328729354099778e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1187, + "tokens/total": 17921896, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 383740 + }, + { + "epoch": 18.5625, + "grad_norm": 8.31157376524061e-05, + "learning_rate": 4.6907762579214744e-05, + "loss": 2.4303824375238037e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1188, + "tokens/total": 17937040, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 384060 + }, + { + "epoch": 18.578125, + "grad_norm": 5.3303658205550164e-05, + "learning_rate": 4.6836310128095015e-05, + "loss": 1.740478637657361e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1189, + "tokens/total": 17952080, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 384381 + }, + { + "epoch": 18.59375, + "grad_norm": 0.00012431015784386545, + "learning_rate": 4.676487895348949e-05, + "loss": 2.4491496333212126e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1190, + "tokens/total": 17967236, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 384704 + }, + { + "epoch": 18.609375, + "grad_norm": 5.924674405832775e-05, + "learning_rate": 4.6693469241564755e-05, + "loss": 2.1186499452596763e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1191, + "tokens/total": 17982240, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 385023 + }, + { + "epoch": 18.625, + "grad_norm": 5.131677244207822e-05, + "learning_rate": 4.662208117843145e-05, + "loss": 1.786351390364871e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1192, + "tokens/total": 17997448, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 385350 + }, + { + "epoch": 18.640625, + "grad_norm": 4.0995666495291516e-05, + "learning_rate": 4.6550714950143846e-05, + "loss": 1.603251462256594e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1193, + "tokens/total": 18012444, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 385676 + }, + { + "epoch": 18.65625, + "grad_norm": 4.146132050664164e-05, + "learning_rate": 4.6479370742699277e-05, + "loss": 1.6708216890037875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1194, + "tokens/total": 18027620, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 386000 + }, + { + "epoch": 18.671875, + "grad_norm": 4.9032631068257615e-05, + "learning_rate": 4.640804874203769e-05, + "loss": 1.6799722288851626e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1195, + "tokens/total": 18042704, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 386324 + }, + { + "epoch": 18.6875, + "grad_norm": 6.009405478835106e-05, + "learning_rate": 4.633674913404112e-05, + "loss": 1.8986443137691822e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1196, + "tokens/total": 18057756, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 386648 + }, + { + "epoch": 18.703125, + "grad_norm": 0.00010683456639526412, + "learning_rate": 4.626547210453333e-05, + "loss": 2.2277286007010844e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.01, + "memory/max_allocated (GiB)": 33.01, + "ppl": 1.0, + "step": 1197, + "tokens/total": 18072840, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 386971 + }, + { + "epoch": 18.71875, + "grad_norm": 7.208803435787559e-05, + "learning_rate": 4.619421783927913e-05, + "loss": 1.779767444531899e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1198, + "tokens/total": 18087852, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 387292 + }, + { + "epoch": 18.734375, + "grad_norm": 5.5161057389341295e-05, + "learning_rate": 4.61229865239841e-05, + "loss": 2.0354655134724453e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1199, + "tokens/total": 18102908, + "tokens/train_per_sec_per_gpu": 10.13, + "tokens/trainable": 387617 + }, + { + "epoch": 18.75, + "grad_norm": 3.943093906855211e-05, + "learning_rate": 4.6051778344293914e-05, + "loss": 1.5674011137889465e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1200, + "tokens/total": 18117924, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 387939 + }, + { + "epoch": 18.765625, + "grad_norm": 9.006608161143959e-05, + "learning_rate": 4.598059348579402e-05, + "loss": 2.0626730474759825e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1201, + "tokens/total": 18133012, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 388262 + }, + { + "epoch": 18.78125, + "grad_norm": 4.72897918371018e-05, + "learning_rate": 4.590943213400907e-05, + "loss": 1.6321798739227233e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1202, + "tokens/total": 18148140, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 388585 + }, + { + "epoch": 18.796875, + "grad_norm": 3.705505150719546e-05, + "learning_rate": 4.583829447440244e-05, + "loss": 1.4403276509256102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1203, + "tokens/total": 18163272, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 388910 + }, + { + "epoch": 18.8125, + "grad_norm": 4.301040098653175e-05, + "learning_rate": 4.576718069237575e-05, + "loss": 1.4911016705809743e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1204, + "tokens/total": 18178428, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 389238 + }, + { + "epoch": 18.828125, + "grad_norm": 4.895324309472926e-05, + "learning_rate": 4.56960909732684e-05, + "loss": 1.442468033019395e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1205, + "tokens/total": 18193456, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 389563 + }, + { + "epoch": 18.84375, + "grad_norm": 5.361656440072693e-05, + "learning_rate": 4.562502550235709e-05, + "loss": 1.7629415651754243e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1206, + "tokens/total": 18208416, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 389886 + }, + { + "epoch": 18.859375, + "grad_norm": 3.772081254282966e-05, + "learning_rate": 4.555398446485533e-05, + "loss": 1.505468276263855e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1207, + "tokens/total": 18223516, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 390207 + }, + { + "epoch": 18.875, + "grad_norm": 5.2001945732627064e-05, + "learning_rate": 4.548296804591287e-05, + "loss": 1.7424607676730375e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1208, + "tokens/total": 18238636, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 390529 + }, + { + "epoch": 18.890625, + "grad_norm": 5.329879422788508e-05, + "learning_rate": 4.541197643061541e-05, + "loss": 1.875185716926353e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1209, + "tokens/total": 18253796, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 390857 + }, + { + "epoch": 18.90625, + "grad_norm": 4.5940567360958084e-05, + "learning_rate": 4.5341009803983924e-05, + "loss": 1.6320434497174574e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1210, + "tokens/total": 18268888, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 391180 + }, + { + "epoch": 18.921875, + "grad_norm": 3.797456156462431e-05, + "learning_rate": 4.527006835097435e-05, + "loss": 1.3763614106210298e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1211, + "tokens/total": 18284088, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 391501 + }, + { + "epoch": 18.9375, + "grad_norm": 4.424414146342315e-05, + "learning_rate": 4.5199152256476895e-05, + "loss": 1.6099708091132925e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1212, + "tokens/total": 18299152, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 391822 + }, + { + "epoch": 18.953125, + "grad_norm": 4.7647819883422926e-05, + "learning_rate": 4.512826170531577e-05, + "loss": 1.7502624132248457e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1213, + "tokens/total": 18314300, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 392146 + }, + { + "epoch": 18.96875, + "grad_norm": 3.070956881856546e-05, + "learning_rate": 4.505739688224856e-05, + "loss": 1.1373566621841746e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1214, + "tokens/total": 18329512, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 392470 + }, + { + "epoch": 18.984375, + "grad_norm": 0.00014209213259164244, + "learning_rate": 4.498655797196586e-05, + "loss": 2.0515483356575714e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1215, + "tokens/total": 18344580, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 392791 + }, + { + "epoch": 19.0, + "grad_norm": 8.708070527063683e-05, + "learning_rate": 4.491574515909066e-05, + "loss": 1.7280616475545685e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1216, + "tokens/total": 18359720, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 393110 + }, + { + "epoch": 19.015625, + "grad_norm": 5.036526272306219e-05, + "learning_rate": 4.484495862817796e-05, + "loss": 1.5402714552692487e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1217, + "tokens/total": 18374876, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 393431 + }, + { + "epoch": 19.03125, + "grad_norm": 3.763810309465043e-05, + "learning_rate": 4.4774198563714274e-05, + "loss": 1.2979957091374672e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1218, + "tokens/total": 18389976, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 393755 + }, + { + "epoch": 19.046875, + "grad_norm": 3.832763104583137e-05, + "learning_rate": 4.470346515011716e-05, + "loss": 1.2925370356242638e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1219, + "tokens/total": 18405156, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 394077 + }, + { + "epoch": 19.0625, + "grad_norm": 7.714011735515669e-05, + "learning_rate": 4.4632758571734645e-05, + "loss": 1.8202929368271725e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1220, + "tokens/total": 18420180, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 394397 + }, + { + "epoch": 19.078125, + "grad_norm": 4.047526454087347e-05, + "learning_rate": 4.456207901284488e-05, + "loss": 1.3020320466239355e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1221, + "tokens/total": 18435268, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 394714 + }, + { + "epoch": 19.09375, + "grad_norm": 3.734684651135467e-05, + "learning_rate": 4.44914266576556e-05, + "loss": 1.3558432101490325e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1222, + "tokens/total": 18450336, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 395039 + }, + { + "epoch": 19.109375, + "grad_norm": 3.928831938537769e-05, + "learning_rate": 4.442080169030356e-05, + "loss": 1.4763646731807967e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1223, + "tokens/total": 18465452, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 395362 + }, + { + "epoch": 19.125, + "grad_norm": 3.848071355605498e-05, + "learning_rate": 4.435020429485425e-05, + "loss": 1.2980127621631254e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1224, + "tokens/total": 18480552, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 395690 + }, + { + "epoch": 19.140625, + "grad_norm": 5.0784292398020625e-05, + "learning_rate": 4.427963465530121e-05, + "loss": 1.7075694813684095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1225, + "tokens/total": 18495576, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 396014 + }, + { + "epoch": 19.15625, + "grad_norm": 0.00012907122436445206, + "learning_rate": 4.420909295556571e-05, + "loss": 1.944498308148468e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1226, + "tokens/total": 18510732, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 396336 + }, + { + "epoch": 19.171875, + "grad_norm": 4.278722917661071e-05, + "learning_rate": 4.413857937949614e-05, + "loss": 1.4317242857941892e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1227, + "tokens/total": 18525832, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 396660 + }, + { + "epoch": 19.1875, + "grad_norm": 0.0001286675251321867, + "learning_rate": 4.406809411086763e-05, + "loss": 1.567839490235201e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1228, + "tokens/total": 18540872, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 396983 + }, + { + "epoch": 19.203125, + "grad_norm": 4.617152808350511e-05, + "learning_rate": 4.3997637333381524e-05, + "loss": 1.4183483472152147e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1229, + "tokens/total": 18555872, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 397307 + }, + { + "epoch": 19.21875, + "grad_norm": 0.00020694639533758163, + "learning_rate": 4.3927209230664934e-05, + "loss": 2.028954440902453e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1230, + "tokens/total": 18571032, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 397633 + }, + { + "epoch": 19.234375, + "grad_norm": 5.134443563292734e-05, + "learning_rate": 4.385680998627019e-05, + "loss": 1.4353051938087447e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1231, + "tokens/total": 18586096, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 397957 + }, + { + "epoch": 19.25, + "grad_norm": 6.167407991597429e-05, + "learning_rate": 4.378643978367445e-05, + "loss": 1.511565756118216e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1232, + "tokens/total": 18601160, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 398280 + }, + { + "epoch": 19.265625, + "grad_norm": 3.5731063690036535e-05, + "learning_rate": 4.371609880627917e-05, + "loss": 1.305771775150788e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1233, + "tokens/total": 18616308, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 398604 + }, + { + "epoch": 19.28125, + "grad_norm": 0.00027790680178441107, + "learning_rate": 4.364578723740964e-05, + "loss": 4.581113898893818e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1234, + "tokens/total": 18631468, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 398927 + }, + { + "epoch": 19.296875, + "grad_norm": 3.7459270970430225e-05, + "learning_rate": 4.3575505260314506e-05, + "loss": 1.49201002841437e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1235, + "tokens/total": 18646552, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 399253 + }, + { + "epoch": 19.3125, + "grad_norm": 3.358343383297324e-05, + "learning_rate": 4.3505253058165275e-05, + "loss": 1.2455773230612976e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1236, + "tokens/total": 18661528, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 399575 + }, + { + "epoch": 19.328125, + "grad_norm": 4.37937123933807e-05, + "learning_rate": 4.3435030814055876e-05, + "loss": 1.6122190800160752e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1237, + "tokens/total": 18676632, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 399900 + }, + { + "epoch": 19.34375, + "grad_norm": 0.00011328987602610141, + "learning_rate": 4.336483871100218e-05, + "loss": 2.8422148261597613e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1238, + "tokens/total": 18691772, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 400224 + }, + { + "epoch": 19.359375, + "grad_norm": 6.13719312241301e-05, + "learning_rate": 4.3294676931941405e-05, + "loss": 1.8224473024019971e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1239, + "tokens/total": 18706884, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 400548 + }, + { + "epoch": 19.375, + "grad_norm": 3.200676655978896e-05, + "learning_rate": 4.3224545659731854e-05, + "loss": 1.1073914265580242e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1240, + "tokens/total": 18722128, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 400873 + }, + { + "epoch": 19.390625, + "grad_norm": 8.10952769825235e-05, + "learning_rate": 4.315444507715226e-05, + "loss": 1.303109911532374e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1241, + "tokens/total": 18737192, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 401199 + }, + { + "epoch": 19.40625, + "grad_norm": 3.800067861448042e-05, + "learning_rate": 4.308437536690142e-05, + "loss": 1.3624684243040974e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1242, + "tokens/total": 18752228, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 401521 + }, + { + "epoch": 19.421875, + "grad_norm": 4.035232632304542e-05, + "learning_rate": 4.3014336711597594e-05, + "loss": 1.431725195288891e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1243, + "tokens/total": 18767256, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 401843 + }, + { + "epoch": 19.4375, + "grad_norm": 4.9487720389151946e-05, + "learning_rate": 4.294432929377814e-05, + "loss": 1.6788558241387364e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1244, + "tokens/total": 18782420, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 402165 + }, + { + "epoch": 19.453125, + "grad_norm": 3.826040847343393e-05, + "learning_rate": 4.287435329589906e-05, + "loss": 1.4663864931208082e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1245, + "tokens/total": 18797532, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 402488 + }, + { + "epoch": 19.46875, + "grad_norm": 6.12035219091922e-05, + "learning_rate": 4.2804408900334394e-05, + "loss": 1.9625676941359416e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1246, + "tokens/total": 18812688, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 402813 + }, + { + "epoch": 19.484375, + "grad_norm": 3.5675497201737016e-05, + "learning_rate": 4.273449628937583e-05, + "loss": 1.3535274092646432e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1247, + "tokens/total": 18827700, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 403136 + }, + { + "epoch": 19.5, + "grad_norm": 3.640030627138913e-05, + "learning_rate": 4.266461564523224e-05, + "loss": 1.3300012824402074e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1248, + "tokens/total": 18842784, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 403463 + }, + { + "epoch": 19.515625, + "grad_norm": 0.00012633735605049878, + "learning_rate": 4.259476715002919e-05, + "loss": 2.720780230447417e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1249, + "tokens/total": 18857880, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 403786 + }, + { + "epoch": 19.53125, + "grad_norm": 6.185249367263168e-05, + "learning_rate": 4.2524950985808433e-05, + "loss": 1.7444958757550921e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1250, + "tokens/total": 18873028, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 404115 + }, + { + "epoch": 19.546875, + "grad_norm": 3.4217024222016335e-05, + "learning_rate": 4.2455167334527466e-05, + "loss": 1.383493668072333e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1251, + "tokens/total": 18888056, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 404440 + }, + { + "epoch": 19.5625, + "grad_norm": 4.467298640520312e-05, + "learning_rate": 4.238541637805905e-05, + "loss": 1.5995102558008512e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1252, + "tokens/total": 18903208, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 404765 + }, + { + "epoch": 19.578125, + "grad_norm": 6.922840111656114e-05, + "learning_rate": 4.231569829819075e-05, + "loss": 2.1741448108514305e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1253, + "tokens/total": 18918404, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 405087 + }, + { + "epoch": 19.59375, + "grad_norm": 3.0983643227955326e-05, + "learning_rate": 4.2246013276624464e-05, + "loss": 1.0555747849139152e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1254, + "tokens/total": 18933532, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 405407 + }, + { + "epoch": 19.609375, + "grad_norm": 3.5063683753833175e-05, + "learning_rate": 4.217636149497584e-05, + "loss": 1.2884529496659525e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1255, + "tokens/total": 18948644, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 405727 + }, + { + "epoch": 19.625, + "grad_norm": 2.3181348296930082e-05, + "learning_rate": 4.210674313477399e-05, + "loss": 8.885665465641068e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1256, + "tokens/total": 18963700, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 406054 + }, + { + "epoch": 19.640625, + "grad_norm": 3.877383278449997e-05, + "learning_rate": 4.203715837746092e-05, + "loss": 1.3883994824936963e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1257, + "tokens/total": 18978812, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 406374 + }, + { + "epoch": 19.65625, + "grad_norm": 5.995816900394857e-05, + "learning_rate": 4.196760740439097e-05, + "loss": 1.895233594950696e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1258, + "tokens/total": 18993824, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 406696 + }, + { + "epoch": 19.671875, + "grad_norm": 4.023635119665414e-05, + "learning_rate": 4.189809039683055e-05, + "loss": 1.533591444058402e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1259, + "tokens/total": 19009044, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 407018 + }, + { + "epoch": 19.6875, + "grad_norm": 3.9723614463582635e-05, + "learning_rate": 4.1828607535957435e-05, + "loss": 1.387587758472364e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1260, + "tokens/total": 19024196, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 407344 + }, + { + "epoch": 19.703125, + "grad_norm": 2.6917889044852927e-05, + "learning_rate": 4.1759159002860504e-05, + "loss": 9.686137900644098e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1261, + "tokens/total": 19039252, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 407668 + }, + { + "epoch": 19.71875, + "grad_norm": 2.3882739697000943e-05, + "learning_rate": 4.168974497853909e-05, + "loss": 8.95195739758492e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1262, + "tokens/total": 19054368, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 407991 + }, + { + "epoch": 19.734375, + "grad_norm": 8.37330735521391e-05, + "learning_rate": 4.162036564390264e-05, + "loss": 2.0238151137164095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1263, + "tokens/total": 19069548, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 408313 + }, + { + "epoch": 19.75, + "grad_norm": 4.303498280933127e-05, + "learning_rate": 4.1551021179770166e-05, + "loss": 1.3718847640120657e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1264, + "tokens/total": 19084724, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 408635 + }, + { + "epoch": 19.765625, + "grad_norm": 3.516512879286893e-05, + "learning_rate": 4.148171176686983e-05, + "loss": 1.3249290304884198e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1265, + "tokens/total": 19099812, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 408957 + }, + { + "epoch": 19.78125, + "grad_norm": 0.00010892192221945152, + "learning_rate": 4.141243758583837e-05, + "loss": 1.7747294123182655e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1266, + "tokens/total": 19114956, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 409276 + }, + { + "epoch": 19.796875, + "grad_norm": 6.204351666383445e-05, + "learning_rate": 4.1343198817220784e-05, + "loss": 1.5592834188282723e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1267, + "tokens/total": 19130024, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 409599 + }, + { + "epoch": 19.8125, + "grad_norm": 3.794864460360259e-05, + "learning_rate": 4.127399564146975e-05, + "loss": 1.4912950518919388e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1268, + "tokens/total": 19145196, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 409923 + }, + { + "epoch": 19.828125, + "grad_norm": 0.0001267164188902825, + "learning_rate": 4.120482823894516e-05, + "loss": 1.5678674571972806e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1269, + "tokens/total": 19160252, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 410246 + }, + { + "epoch": 19.84375, + "grad_norm": 2.4073671738733537e-05, + "learning_rate": 4.113569678991369e-05, + "loss": 8.244655873568263e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1270, + "tokens/total": 19175352, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 410571 + }, + { + "epoch": 19.859375, + "grad_norm": 8.351013093488291e-05, + "learning_rate": 4.106660147454832e-05, + "loss": 1.6358908396796323e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1271, + "tokens/total": 19190464, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 410890 + }, + { + "epoch": 19.875, + "grad_norm": 0.0001255474635399878, + "learning_rate": 4.099754247292782e-05, + "loss": 2.4098144422168843e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1272, + "tokens/total": 19205592, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 411217 + }, + { + "epoch": 19.890625, + "grad_norm": 9.613324073143303e-05, + "learning_rate": 4.0928519965036385e-05, + "loss": 2.000823542402941e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1273, + "tokens/total": 19220660, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 411541 + }, + { + "epoch": 19.90625, + "grad_norm": 5.7355977332917973e-05, + "learning_rate": 4.085953413076302e-05, + "loss": 1.6018575479392894e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1274, + "tokens/total": 19235708, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 411863 + }, + { + "epoch": 19.921875, + "grad_norm": 3.954311250708997e-05, + "learning_rate": 4.079058514990122e-05, + "loss": 1.4924756897016778e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1275, + "tokens/total": 19250720, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 412184 + }, + { + "epoch": 19.9375, + "grad_norm": 5.070498809800483e-05, + "learning_rate": 4.072167320214838e-05, + "loss": 1.4584103382730973e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1276, + "tokens/total": 19265824, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 412505 + }, + { + "epoch": 19.953125, + "grad_norm": 5.1608109060907736e-05, + "learning_rate": 4.0652798467105425e-05, + "loss": 1.3493569213096634e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1277, + "tokens/total": 19280920, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 412831 + }, + { + "epoch": 19.96875, + "grad_norm": 4.347148569650017e-05, + "learning_rate": 4.058396112427622e-05, + "loss": 1.481978983974841e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1278, + "tokens/total": 19295996, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 413157 + }, + { + "epoch": 19.984375, + "grad_norm": 5.026731378166005e-05, + "learning_rate": 4.0515161353067274e-05, + "loss": 1.598478775122203e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1279, + "tokens/total": 19311200, + "tokens/train_per_sec_per_gpu": 8.64, + "tokens/trainable": 413475 + }, + { + "epoch": 20.0, + "grad_norm": 6.949734961381182e-05, + "learning_rate": 4.04463993327871e-05, + "loss": 1.941732307386701e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1280, + "tokens/total": 19326268, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 413800 + }, + { + "epoch": 20.015625, + "grad_norm": 3.829331399174407e-05, + "learning_rate": 4.037767524264587e-05, + "loss": 1.2854493434133474e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1281, + "tokens/total": 19341372, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 414124 + }, + { + "epoch": 20.03125, + "grad_norm": 5.847791180713102e-05, + "learning_rate": 4.030898926175486e-05, + "loss": 1.5375832163044834e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1282, + "tokens/total": 19356556, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 414449 + }, + { + "epoch": 20.046875, + "grad_norm": 3.0854163924232125e-05, + "learning_rate": 4.024034156912605e-05, + "loss": 1.148289697994187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1283, + "tokens/total": 19371600, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 414767 + }, + { + "epoch": 20.0625, + "grad_norm": 3.65119194611907e-05, + "learning_rate": 4.017173234367164e-05, + "loss": 1.4035761068953434e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1284, + "tokens/total": 19386784, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 415088 + }, + { + "epoch": 20.078125, + "grad_norm": 4.0353632357437164e-05, + "learning_rate": 4.0103161764203546e-05, + "loss": 1.4293543699750444e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1285, + "tokens/total": 19401948, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 415416 + }, + { + "epoch": 20.09375, + "grad_norm": 3.269499211455695e-05, + "learning_rate": 4.0034630009432986e-05, + "loss": 1.2137394378441968e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1286, + "tokens/total": 19417056, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 415738 + }, + { + "epoch": 20.109375, + "grad_norm": 2.50589528150158e-05, + "learning_rate": 3.996613725796998e-05, + "loss": 9.470993518334581e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1287, + "tokens/total": 19432244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 416064 + }, + { + "epoch": 20.125, + "grad_norm": 3.315035792184062e-05, + "learning_rate": 3.989768368832291e-05, + "loss": 1.176838395622326e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1288, + "tokens/total": 19447376, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 416388 + }, + { + "epoch": 20.140625, + "grad_norm": 2.593081080703996e-05, + "learning_rate": 3.9829269478898026e-05, + "loss": 9.639439895181567e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1289, + "tokens/total": 19462444, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 416712 + }, + { + "epoch": 20.15625, + "grad_norm": 8.535145752830431e-05, + "learning_rate": 3.9760894807999004e-05, + "loss": 1.7392931113136e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1290, + "tokens/total": 19477532, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 417038 + }, + { + "epoch": 20.171875, + "grad_norm": 6.821544229751453e-05, + "learning_rate": 3.969255985382646e-05, + "loss": 1.538321271254972e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1291, + "tokens/total": 19492660, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 417359 + }, + { + "epoch": 20.1875, + "grad_norm": 2.6004734536400065e-05, + "learning_rate": 3.9624264794477515e-05, + "loss": 8.986547754830099e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1292, + "tokens/total": 19507680, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 417679 + }, + { + "epoch": 20.203125, + "grad_norm": 3.369764090166427e-05, + "learning_rate": 3.9556009807945335e-05, + "loss": 1.188762780657271e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1293, + "tokens/total": 19522760, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 418003 + }, + { + "epoch": 20.21875, + "grad_norm": 4.5328924898058176e-05, + "learning_rate": 3.948779507211859e-05, + "loss": 1.4277691207098542e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1294, + "tokens/total": 19537868, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 418329 + }, + { + "epoch": 20.234375, + "grad_norm": 3.688072683871724e-05, + "learning_rate": 3.9419620764781094e-05, + "loss": 1.2887861657873145e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1295, + "tokens/total": 19552928, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 418652 + }, + { + "epoch": 20.25, + "grad_norm": 3.3802702091634274e-05, + "learning_rate": 3.935148706361128e-05, + "loss": 1.123045194617589e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1296, + "tokens/total": 19568036, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 418972 + }, + { + "epoch": 20.265625, + "grad_norm": 2.8554604796227068e-05, + "learning_rate": 3.928339414618174e-05, + "loss": 1.0779156127682654e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1297, + "tokens/total": 19583140, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 419298 + }, + { + "epoch": 20.28125, + "grad_norm": 9.431726357433945e-05, + "learning_rate": 3.921534218995878e-05, + "loss": 1.0807627859321656e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1298, + "tokens/total": 19598276, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 419618 + }, + { + "epoch": 20.296875, + "grad_norm": 0.00023360468912869692, + "learning_rate": 3.914733137230198e-05, + "loss": 1.1309866749797948e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1299, + "tokens/total": 19613396, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 419939 + }, + { + "epoch": 20.3125, + "grad_norm": 9.108100493904203e-05, + "learning_rate": 3.907936187046366e-05, + "loss": 1.720442242003628e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1300, + "tokens/total": 19628456, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 420263 + }, + { + "epoch": 20.328125, + "grad_norm": 4.4541822717292234e-05, + "learning_rate": 3.9011433861588484e-05, + "loss": 1.5672435438318644e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1301, + "tokens/total": 19643552, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 420585 + }, + { + "epoch": 20.34375, + "grad_norm": 3.399713023100048e-05, + "learning_rate": 3.8943547522712976e-05, + "loss": 1.2560042250697734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1302, + "tokens/total": 19658608, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 420910 + }, + { + "epoch": 20.359375, + "grad_norm": 0.00013420746836345643, + "learning_rate": 3.8875703030765056e-05, + "loss": 1.7695567748887697e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1303, + "tokens/total": 19673628, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 421233 + }, + { + "epoch": 20.375, + "grad_norm": 3.663083407445811e-05, + "learning_rate": 3.880790056256357e-05, + "loss": 1.139798655458435e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1304, + "tokens/total": 19688680, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 421558 + }, + { + "epoch": 20.390625, + "grad_norm": 3.142326750094071e-05, + "learning_rate": 3.874014029481785e-05, + "loss": 1.0110304629051825e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1305, + "tokens/total": 19703752, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 421887 + }, + { + "epoch": 20.40625, + "grad_norm": 4.316388367442414e-05, + "learning_rate": 3.867242240412725e-05, + "loss": 1.459539248571673e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1306, + "tokens/total": 19718836, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 422209 + }, + { + "epoch": 20.421875, + "grad_norm": 3.6985678889323026e-05, + "learning_rate": 3.860474706698067e-05, + "loss": 1.3256059219202143e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1307, + "tokens/total": 19733936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 422535 + }, + { + "epoch": 20.4375, + "grad_norm": 0.00011261684994678944, + "learning_rate": 3.8537114459756106e-05, + "loss": 1.3524604582926258e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1308, + "tokens/total": 19749040, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 422864 + }, + { + "epoch": 20.453125, + "grad_norm": 0.00010908506374107674, + "learning_rate": 3.84695247587202e-05, + "loss": 1.936456783369067e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1309, + "tokens/total": 19764032, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 423186 + }, + { + "epoch": 20.46875, + "grad_norm": 3.795835800701752e-05, + "learning_rate": 3.840197814002774e-05, + "loss": 1.3547804655900109e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1310, + "tokens/total": 19779116, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 423505 + }, + { + "epoch": 20.484375, + "grad_norm": 2.9260363589855842e-05, + "learning_rate": 3.8334474779721264e-05, + "loss": 1.074531155609293e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1311, + "tokens/total": 19794200, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 423828 + }, + { + "epoch": 20.5, + "grad_norm": 6.67829517624341e-05, + "learning_rate": 3.826701485373056e-05, + "loss": 1.6409317140642088e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1312, + "tokens/total": 19809224, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 424147 + }, + { + "epoch": 20.515625, + "grad_norm": 4.2259478505002335e-05, + "learning_rate": 3.819959853787222e-05, + "loss": 1.245134058081021e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1313, + "tokens/total": 19824340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 424469 + }, + { + "epoch": 20.53125, + "grad_norm": 0.00015263629029504955, + "learning_rate": 3.813222600784915e-05, + "loss": 1.4897166238370119e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1314, + "tokens/total": 19839468, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 424792 + }, + { + "epoch": 20.546875, + "grad_norm": 3.405763709452003e-05, + "learning_rate": 3.806489743925016e-05, + "loss": 1.1253252978349337e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1315, + "tokens/total": 19854660, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 425113 + }, + { + "epoch": 20.5625, + "grad_norm": 3.544608989614062e-05, + "learning_rate": 3.799761300754953e-05, + "loss": 1.315873078056029e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1316, + "tokens/total": 19869740, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 425433 + }, + { + "epoch": 20.578125, + "grad_norm": 4.771173553308472e-05, + "learning_rate": 3.7930372888106404e-05, + "loss": 1.3652689858645317e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1317, + "tokens/total": 19884888, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 425758 + }, + { + "epoch": 20.59375, + "grad_norm": 2.4320219381479546e-05, + "learning_rate": 3.7863177256164526e-05, + "loss": 8.742082400203799e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1318, + "tokens/total": 19900020, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 426084 + }, + { + "epoch": 20.609375, + "grad_norm": 3.2074829505290836e-05, + "learning_rate": 3.779602628685167e-05, + "loss": 1.1671085076159216e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1319, + "tokens/total": 19915184, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 426408 + }, + { + "epoch": 20.625, + "grad_norm": 4.1550043533788994e-05, + "learning_rate": 3.772892015517924e-05, + "loss": 1.4509197399092955e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1320, + "tokens/total": 19930336, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 426731 + }, + { + "epoch": 20.640625, + "grad_norm": 3.336342342663556e-05, + "learning_rate": 3.7661859036041666e-05, + "loss": 1.2546064453999861e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1321, + "tokens/total": 19945444, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 427049 + }, + { + "epoch": 20.65625, + "grad_norm": 0.00020840902288910002, + "learning_rate": 3.7594843104216215e-05, + "loss": 3.5690584354597377e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1322, + "tokens/total": 19960440, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 427374 + }, + { + "epoch": 20.671875, + "grad_norm": 3.13690907205455e-05, + "learning_rate": 3.752787253436231e-05, + "loss": 1.094694539460761e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1323, + "tokens/total": 19975548, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 427699 + }, + { + "epoch": 20.6875, + "grad_norm": 3.192544318153523e-05, + "learning_rate": 3.746094750102113e-05, + "loss": 1.1861247912747785e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1324, + "tokens/total": 19990704, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 428020 + }, + { + "epoch": 20.703125, + "grad_norm": 3.4285360015928745e-05, + "learning_rate": 3.739406817861523e-05, + "loss": 1.2080217857146636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1325, + "tokens/total": 20005780, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 428341 + }, + { + "epoch": 20.71875, + "grad_norm": 2.0670053345384076e-05, + "learning_rate": 3.732723474144797e-05, + "loss": 7.699321145082649e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1326, + "tokens/total": 20020948, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 428661 + }, + { + "epoch": 20.734375, + "grad_norm": 8.919210085878149e-05, + "learning_rate": 3.726044736370319e-05, + "loss": 1.0944050927719218e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1327, + "tokens/total": 20035924, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 428982 + }, + { + "epoch": 20.75, + "grad_norm": 4.0302569686900824e-05, + "learning_rate": 3.7193706219444647e-05, + "loss": 1.18612172173016e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1328, + "tokens/total": 20051084, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 429312 + }, + { + "epoch": 20.765625, + "grad_norm": 9.667545964475721e-05, + "learning_rate": 3.712701148261559e-05, + "loss": 2.346309656786616e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1329, + "tokens/total": 20066216, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 429641 + }, + { + "epoch": 20.78125, + "grad_norm": 4.990700836060569e-05, + "learning_rate": 3.706036332703836e-05, + "loss": 1.3072736919639283e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1330, + "tokens/total": 20081184, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 429963 + }, + { + "epoch": 20.796875, + "grad_norm": 2.896969272114802e-05, + "learning_rate": 3.6993761926413885e-05, + "loss": 1.0985164635712863e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1331, + "tokens/total": 20096188, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 430286 + }, + { + "epoch": 20.8125, + "grad_norm": 8.32487567095086e-05, + "learning_rate": 3.692720745432119e-05, + "loss": 1.7211066278832732e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1332, + "tokens/total": 20111212, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 430609 + }, + { + "epoch": 20.828125, + "grad_norm": 5.533869261853397e-05, + "learning_rate": 3.686070008421706e-05, + "loss": 1.4217890793588595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1333, + "tokens/total": 20126416, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 430933 + }, + { + "epoch": 20.84375, + "grad_norm": 3.13527270918712e-05, + "learning_rate": 3.6794239989435483e-05, + "loss": 1.147724105976522e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1334, + "tokens/total": 20141480, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 431258 + }, + { + "epoch": 20.859375, + "grad_norm": 6.830783240729943e-05, + "learning_rate": 3.672782734318728e-05, + "loss": 1.7105634242398082e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1335, + "tokens/total": 20156568, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 431582 + }, + { + "epoch": 20.875, + "grad_norm": 6.85417908243835e-05, + "learning_rate": 3.666146231855953e-05, + "loss": 1.7634728237680974e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1336, + "tokens/total": 20171636, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 431909 + }, + { + "epoch": 20.890625, + "grad_norm": 4.207641177345067e-05, + "learning_rate": 3.659514508851525e-05, + "loss": 1.302434043282119e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1337, + "tokens/total": 20186768, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 432232 + }, + { + "epoch": 20.90625, + "grad_norm": 0.00012472961680032313, + "learning_rate": 3.6528875825892896e-05, + "loss": 2.4291086901939707e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1338, + "tokens/total": 20201984, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 432557 + }, + { + "epoch": 20.921875, + "grad_norm": 3.0562354368157685e-05, + "learning_rate": 3.646265470340593e-05, + "loss": 1.0717330951592885e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1339, + "tokens/total": 20217052, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 432880 + }, + { + "epoch": 20.9375, + "grad_norm": 4.532597085926682e-05, + "learning_rate": 3.6396481893642274e-05, + "loss": 1.3894569974581827e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1340, + "tokens/total": 20232188, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 433199 + }, + { + "epoch": 20.953125, + "grad_norm": 8.60459913383238e-05, + "learning_rate": 3.633035756906401e-05, + "loss": 1.998113930312684e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1341, + "tokens/total": 20247212, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 433526 + }, + { + "epoch": 20.96875, + "grad_norm": 4.393285416881554e-05, + "learning_rate": 3.626428190200683e-05, + "loss": 1.535434421384707e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1342, + "tokens/total": 20262240, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 433848 + }, + { + "epoch": 20.984375, + "grad_norm": 7.683681178605184e-05, + "learning_rate": 3.619825506467961e-05, + "loss": 1.6070564470282989e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1343, + "tokens/total": 20277504, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 434171 + }, + { + "epoch": 21.0, + "grad_norm": 3.11611765937414e-05, + "learning_rate": 3.6132277229163934e-05, + "loss": 1.1712110108419438e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1344, + "tokens/total": 20292592, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 434490 + }, + { + "epoch": 21.015625, + "grad_norm": 5.231054819887504e-05, + "learning_rate": 3.6066348567413746e-05, + "loss": 1.4133705690255738e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1345, + "tokens/total": 20307648, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 434816 + }, + { + "epoch": 21.03125, + "grad_norm": 2.8977727197343484e-05, + "learning_rate": 3.6000469251254764e-05, + "loss": 1.1185899211341166e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1346, + "tokens/total": 20322720, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 435137 + }, + { + "epoch": 21.046875, + "grad_norm": 6.262312672333792e-05, + "learning_rate": 3.5934639452384176e-05, + "loss": 1.6863264136190992e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1347, + "tokens/total": 20337848, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 435463 + }, + { + "epoch": 21.0625, + "grad_norm": 3.341737829032354e-05, + "learning_rate": 3.586885934237001e-05, + "loss": 1.1420321470723138e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1348, + "tokens/total": 20353052, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 435779 + }, + { + "epoch": 21.078125, + "grad_norm": 4.1886622057063505e-05, + "learning_rate": 3.580312909265088e-05, + "loss": 1.2238497220096178e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1349, + "tokens/total": 20368052, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 436101 + }, + { + "epoch": 21.09375, + "grad_norm": 2.773126652755309e-05, + "learning_rate": 3.573744887453543e-05, + "loss": 9.987027169700013e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1350, + "tokens/total": 20383192, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 436423 + }, + { + "epoch": 21.109375, + "grad_norm": 3.3034182706614956e-05, + "learning_rate": 3.5671818859201886e-05, + "loss": 1.0976073099300265e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1351, + "tokens/total": 20398284, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 436746 + }, + { + "epoch": 21.125, + "grad_norm": 2.645220774866175e-05, + "learning_rate": 3.5606239217697643e-05, + "loss": 8.783421208136133e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1352, + "tokens/total": 20413336, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 437072 + }, + { + "epoch": 21.140625, + "grad_norm": 3.607620601542294e-05, + "learning_rate": 3.554071012093886e-05, + "loss": 1.15889088192489e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1353, + "tokens/total": 20428516, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 437394 + }, + { + "epoch": 21.15625, + "grad_norm": 3.3412543416488916e-05, + "learning_rate": 3.547523173970989e-05, + "loss": 1.020706008603156e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1354, + "tokens/total": 20443676, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 437717 + }, + { + "epoch": 21.171875, + "grad_norm": 2.704974940570537e-05, + "learning_rate": 3.5409804244662945e-05, + "loss": 9.035072139340627e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1355, + "tokens/total": 20458780, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 438046 + }, + { + "epoch": 21.1875, + "grad_norm": 2.4573519112891518e-05, + "learning_rate": 3.5344427806317626e-05, + "loss": 8.861654237080074e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1356, + "tokens/total": 20473920, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 438374 + }, + { + "epoch": 21.203125, + "grad_norm": 3.879648647853173e-05, + "learning_rate": 3.527910259506041e-05, + "loss": 1.0080215133712045e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1357, + "tokens/total": 20489016, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 438699 + }, + { + "epoch": 21.21875, + "grad_norm": 7.681656279601157e-05, + "learning_rate": 3.521382878114438e-05, + "loss": 1.6624356931060902e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1358, + "tokens/total": 20504116, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 439026 + }, + { + "epoch": 21.234375, + "grad_norm": 3.8187819882296026e-05, + "learning_rate": 3.514860653468853e-05, + "loss": 1.2353517604424269e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1359, + "tokens/total": 20519248, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 439354 + }, + { + "epoch": 21.25, + "grad_norm": 3.786943125305697e-05, + "learning_rate": 3.508343602567756e-05, + "loss": 1.2698561704382882e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1360, + "tokens/total": 20534400, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 439677 + }, + { + "epoch": 21.265625, + "grad_norm": 2.841778587026056e-05, + "learning_rate": 3.501831742396126e-05, + "loss": 1.0037994115918991e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1361, + "tokens/total": 20549576, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 440001 + }, + { + "epoch": 21.28125, + "grad_norm": 4.843583519686945e-05, + "learning_rate": 3.495325089925419e-05, + "loss": 1.2181541251266026e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1362, + "tokens/total": 20564652, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 440326 + }, + { + "epoch": 21.296875, + "grad_norm": 8.204697951441631e-05, + "learning_rate": 3.488823662113516e-05, + "loss": 1.8879782146541402e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1363, + "tokens/total": 20579792, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 440651 + }, + { + "epoch": 21.3125, + "grad_norm": 4.940873623127118e-05, + "learning_rate": 3.482327475904682e-05, + "loss": 1.576347358422936e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1364, + "tokens/total": 20594872, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 440977 + }, + { + "epoch": 21.328125, + "grad_norm": 3.265003761043772e-05, + "learning_rate": 3.475836548229518e-05, + "loss": 1.0418572173875873e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1365, + "tokens/total": 20610000, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 441300 + }, + { + "epoch": 21.34375, + "grad_norm": 2.9500721211661585e-05, + "learning_rate": 3.469350896004928e-05, + "loss": 1.0471346740814624e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1366, + "tokens/total": 20625080, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 441620 + }, + { + "epoch": 21.359375, + "grad_norm": 3.8911653973627836e-05, + "learning_rate": 3.4628705361340545e-05, + "loss": 1.195441427626065e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1367, + "tokens/total": 20640196, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 441941 + }, + { + "epoch": 21.375, + "grad_norm": 5.0146456487709656e-05, + "learning_rate": 3.45639548550626e-05, + "loss": 1.5920938949420815e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1368, + "tokens/total": 20655112, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 442264 + }, + { + "epoch": 21.390625, + "grad_norm": 3.561767880455591e-05, + "learning_rate": 3.449925760997062e-05, + "loss": 1.2360408163658576e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1369, + "tokens/total": 20670272, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 442584 + }, + { + "epoch": 21.40625, + "grad_norm": 6.971078255446628e-05, + "learning_rate": 3.443461379468098e-05, + "loss": 1.496620257057657e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1370, + "tokens/total": 20685424, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 442904 + }, + { + "epoch": 21.421875, + "grad_norm": 3.373666913830675e-05, + "learning_rate": 3.437002357767079e-05, + "loss": 1.1495437775010942e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1371, + "tokens/total": 20700392, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 443226 + }, + { + "epoch": 21.4375, + "grad_norm": 3.101321271969937e-05, + "learning_rate": 3.430548712727754e-05, + "loss": 1.1348264479238424e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1372, + "tokens/total": 20715468, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 443549 + }, + { + "epoch": 21.453125, + "grad_norm": 3.449022915447131e-05, + "learning_rate": 3.424100461169853e-05, + "loss": 9.408655614606687e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1373, + "tokens/total": 20730692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 443871 + }, + { + "epoch": 21.46875, + "grad_norm": 0.00011563514271983877, + "learning_rate": 3.4176576198990484e-05, + "loss": 1.3544213288696483e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1374, + "tokens/total": 20745688, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 444194 + }, + { + "epoch": 21.484375, + "grad_norm": 4.236880340613425e-05, + "learning_rate": 3.4112202057069164e-05, + "loss": 1.3417014770311653e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1375, + "tokens/total": 20760792, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 444514 + }, + { + "epoch": 21.5, + "grad_norm": 3.213001764379442e-05, + "learning_rate": 3.404788235370884e-05, + "loss": 1.1091860869782977e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1376, + "tokens/total": 20776004, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 444839 + }, + { + "epoch": 21.515625, + "grad_norm": 4.829018871532753e-05, + "learning_rate": 3.398361725654198e-05, + "loss": 1.0469128710610676e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1377, + "tokens/total": 20791076, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 445164 + }, + { + "epoch": 21.53125, + "grad_norm": 2.581994340289384e-05, + "learning_rate": 3.3919406933058646e-05, + "loss": 9.780561640582164e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1378, + "tokens/total": 20806216, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 445488 + }, + { + "epoch": 21.546875, + "grad_norm": 5.570923531195149e-05, + "learning_rate": 3.3855251550606205e-05, + "loss": 1.1661359167192131e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1379, + "tokens/total": 20821228, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 445810 + }, + { + "epoch": 21.5625, + "grad_norm": 2.1066296540084295e-05, + "learning_rate": 3.379115127638879e-05, + "loss": 7.634329790562333e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1380, + "tokens/total": 20836312, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 446130 + }, + { + "epoch": 21.578125, + "grad_norm": 9.813915676204488e-05, + "learning_rate": 3.3727106277466974e-05, + "loss": 2.149949978047516e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1381, + "tokens/total": 20851404, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 446458 + }, + { + "epoch": 21.59375, + "grad_norm": 3.636321707745083e-05, + "learning_rate": 3.366311672075722e-05, + "loss": 1.2032977565468173e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1382, + "tokens/total": 20866552, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 446782 + }, + { + "epoch": 21.609375, + "grad_norm": 2.5315623133792542e-05, + "learning_rate": 3.359918277303149e-05, + "loss": 9.145978197011573e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1383, + "tokens/total": 20881692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 447106 + }, + { + "epoch": 21.625, + "grad_norm": 2.905641849793028e-05, + "learning_rate": 3.353530460091683e-05, + "loss": 9.927587143465644e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1384, + "tokens/total": 20896784, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 447429 + }, + { + "epoch": 21.640625, + "grad_norm": 9.036084520630538e-05, + "learning_rate": 3.3471482370894955e-05, + "loss": 2.1352329895307776e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1385, + "tokens/total": 20911884, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 447751 + }, + { + "epoch": 21.65625, + "grad_norm": 9.428301564184949e-05, + "learning_rate": 3.340771624930173e-05, + "loss": 1.381691959068121e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1386, + "tokens/total": 20927016, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 448076 + }, + { + "epoch": 21.671875, + "grad_norm": 0.00019298357074148953, + "learning_rate": 3.3344006402326814e-05, + "loss": 1.0222339597021346e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1387, + "tokens/total": 20942124, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 448400 + }, + { + "epoch": 21.6875, + "grad_norm": 0.00014079443644732237, + "learning_rate": 3.3280352996013184e-05, + "loss": 1.8012577811532537e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1388, + "tokens/total": 20957224, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 448724 + }, + { + "epoch": 21.703125, + "grad_norm": 2.954769843199756e-05, + "learning_rate": 3.3216756196256724e-05, + "loss": 9.812332564251847e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1389, + "tokens/total": 20972224, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 449047 + }, + { + "epoch": 21.71875, + "grad_norm": 3.4903867344837636e-05, + "learning_rate": 3.315321616880582e-05, + "loss": 1.2507581459431094e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1390, + "tokens/total": 20987256, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 449366 + }, + { + "epoch": 21.734375, + "grad_norm": 5.731922647100873e-05, + "learning_rate": 3.308973307926086e-05, + "loss": 1.4110436268310877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1391, + "tokens/total": 21002392, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 449693 + }, + { + "epoch": 21.75, + "grad_norm": 3.3187432563863695e-05, + "learning_rate": 3.302630709307385e-05, + "loss": 1.2249890914972639e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1392, + "tokens/total": 21017628, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 450013 + }, + { + "epoch": 21.765625, + "grad_norm": 3.2807369279908016e-05, + "learning_rate": 3.2962938375547954e-05, + "loss": 1.1710019407473737e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1393, + "tokens/total": 21032684, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 450333 + }, + { + "epoch": 21.78125, + "grad_norm": 0.00022138067288324237, + "learning_rate": 3.2899627091837135e-05, + "loss": 3.5663420021592174e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1394, + "tokens/total": 21047720, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 450655 + }, + { + "epoch": 21.796875, + "grad_norm": 3.604944868129678e-05, + "learning_rate": 3.283637340694558e-05, + "loss": 1.124550863096374e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1395, + "tokens/total": 21062752, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 450973 + }, + { + "epoch": 21.8125, + "grad_norm": 5.074267028248869e-05, + "learning_rate": 3.277317748572745e-05, + "loss": 1.2324809404162806e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1396, + "tokens/total": 21077856, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 451303 + }, + { + "epoch": 21.828125, + "grad_norm": 5.260012039798312e-05, + "learning_rate": 3.271003949288628e-05, + "loss": 1.284617610508576e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1397, + "tokens/total": 21092976, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 451623 + }, + { + "epoch": 21.84375, + "grad_norm": 3.356956585776061e-05, + "learning_rate": 3.264695959297474e-05, + "loss": 1.1482343325042166e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1398, + "tokens/total": 21108192, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 451953 + }, + { + "epoch": 21.859375, + "grad_norm": 3.087477671215311e-05, + "learning_rate": 3.258393795039395e-05, + "loss": 1.1200238532182993e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1399, + "tokens/total": 21123320, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 452278 + }, + { + "epoch": 21.875, + "grad_norm": 2.0120156477787532e-05, + "learning_rate": 3.252097472939332e-05, + "loss": 7.184031005635916e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1400, + "tokens/total": 21138400, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 452601 + }, + { + "epoch": 21.890625, + "grad_norm": 3.648766505648382e-05, + "learning_rate": 3.2458070094069946e-05, + "loss": 9.755391374710598e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1401, + "tokens/total": 21153508, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 452926 + }, + { + "epoch": 21.90625, + "grad_norm": 8.576009713578969e-05, + "learning_rate": 3.2395224208368245e-05, + "loss": 1.1108490980404895e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1402, + "tokens/total": 21168704, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 453247 + }, + { + "epoch": 21.921875, + "grad_norm": 2.7580974347074516e-05, + "learning_rate": 3.233243723607949e-05, + "loss": 1.0368829634899157e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1403, + "tokens/total": 21183680, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 453565 + }, + { + "epoch": 21.9375, + "grad_norm": 3.5992390621686354e-05, + "learning_rate": 3.226970934084148e-05, + "loss": 1.0626793027768144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1404, + "tokens/total": 21198864, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 453891 + }, + { + "epoch": 21.953125, + "grad_norm": 3.5594166547525674e-05, + "learning_rate": 3.220704068613799e-05, + "loss": 1.1584086223592749e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1405, + "tokens/total": 21213996, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 454212 + }, + { + "epoch": 21.96875, + "grad_norm": 5.1006350986426696e-05, + "learning_rate": 3.2144431435298405e-05, + "loss": 1.4159807051328244e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1406, + "tokens/total": 21229080, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 454535 + }, + { + "epoch": 21.984375, + "grad_norm": 4.2769333958858624e-05, + "learning_rate": 3.20818817514973e-05, + "loss": 1.1573429219424725e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1407, + "tokens/total": 21244184, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 454859 + }, + { + "epoch": 22.0, + "grad_norm": 2.5951305360649712e-05, + "learning_rate": 3.201939179775399e-05, + "loss": 9.776388196769403e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1408, + "tokens/total": 21259228, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 455180 + }, + { + "epoch": 22.015625, + "grad_norm": 3.534326242515817e-05, + "learning_rate": 3.195696173693217e-05, + "loss": 1.0860346719709923e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1409, + "tokens/total": 21274260, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 455504 + }, + { + "epoch": 22.03125, + "grad_norm": 2.904925713664852e-05, + "learning_rate": 3.1894591731739376e-05, + "loss": 9.763703019416425e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1410, + "tokens/total": 21289316, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 455830 + }, + { + "epoch": 22.046875, + "grad_norm": 2.6917170544038527e-05, + "learning_rate": 3.183228194472665e-05, + "loss": 9.31626630062965e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1411, + "tokens/total": 21304508, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 456159 + }, + { + "epoch": 22.0625, + "grad_norm": 2.6051986424135976e-05, + "learning_rate": 3.177003253828808e-05, + "loss": 9.88282977232302e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1412, + "tokens/total": 21319516, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 456481 + }, + { + "epoch": 22.078125, + "grad_norm": 0.00019361668091733009, + "learning_rate": 3.1707843674660454e-05, + "loss": 3.4827728541131364e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1413, + "tokens/total": 21334676, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 456807 + }, + { + "epoch": 22.09375, + "grad_norm": 3.460142033873126e-05, + "learning_rate": 3.164571551592265e-05, + "loss": 1.090206865228538e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1414, + "tokens/total": 21349732, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 457126 + }, + { + "epoch": 22.109375, + "grad_norm": 8.453009650111198e-05, + "learning_rate": 3.1583648223995435e-05, + "loss": 1.9937529032176826e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1415, + "tokens/total": 21364788, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 457448 + }, + { + "epoch": 22.125, + "grad_norm": 3.246117557864636e-05, + "learning_rate": 3.152164196064089e-05, + "loss": 1.1181077752553392e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1416, + "tokens/total": 21379876, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 457773 + }, + { + "epoch": 22.140625, + "grad_norm": 3.3645901567069814e-05, + "learning_rate": 3.1459696887462096e-05, + "loss": 1.0498711162654217e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1417, + "tokens/total": 21395092, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 458101 + }, + { + "epoch": 22.15625, + "grad_norm": 3.1329669582191855e-05, + "learning_rate": 3.139781316590258e-05, + "loss": 1.0679856359274709e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1418, + "tokens/total": 21410224, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 458425 + }, + { + "epoch": 22.171875, + "grad_norm": 2.1550760720856488e-05, + "learning_rate": 3.1335990957246054e-05, + "loss": 8.162268159139785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1419, + "tokens/total": 21425260, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 458749 + }, + { + "epoch": 22.1875, + "grad_norm": 2.2832649847259745e-05, + "learning_rate": 3.127423042261582e-05, + "loss": 8.220714562412468e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1420, + "tokens/total": 21440388, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 459074 + }, + { + "epoch": 22.203125, + "grad_norm": 3.3490825444459915e-05, + "learning_rate": 3.121253172297457e-05, + "loss": 8.721260087440896e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1421, + "tokens/total": 21455424, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 459396 + }, + { + "epoch": 22.21875, + "grad_norm": 3.386957905604504e-05, + "learning_rate": 3.115089501912369e-05, + "loss": 1.1187570407855674e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1422, + "tokens/total": 21470464, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 459711 + }, + { + "epoch": 22.234375, + "grad_norm": 2.6675659682950936e-05, + "learning_rate": 3.108932047170311e-05, + "loss": 9.584093731973553e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1423, + "tokens/total": 21485624, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 460032 + }, + { + "epoch": 22.25, + "grad_norm": 5.678133311448619e-05, + "learning_rate": 3.102780824119071e-05, + "loss": 1.2885598152934108e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1424, + "tokens/total": 21500744, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 460354 + }, + { + "epoch": 22.265625, + "grad_norm": 3.5053599276579916e-05, + "learning_rate": 3.096635848790197e-05, + "loss": 9.70609107753262e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1425, + "tokens/total": 21515860, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 460677 + }, + { + "epoch": 22.28125, + "grad_norm": 3.5153247154084966e-05, + "learning_rate": 3.090497137198953e-05, + "loss": 1.0391299838374835e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1426, + "tokens/total": 21531128, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 461002 + }, + { + "epoch": 22.296875, + "grad_norm": 2.1754487534053624e-05, + "learning_rate": 3.0843647053442784e-05, + "loss": 8.35611672300729e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1427, + "tokens/total": 21546184, + "tokens/train_per_sec_per_gpu": 8.67, + "tokens/trainable": 461326 + }, + { + "epoch": 22.3125, + "grad_norm": 2.4461469365633093e-05, + "learning_rate": 3.078238569208749e-05, + "loss": 8.755021099204896e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1428, + "tokens/total": 21561224, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 461648 + }, + { + "epoch": 22.328125, + "grad_norm": 3.640693103079684e-05, + "learning_rate": 3.072118744758528e-05, + "loss": 1.0287192253599642e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1429, + "tokens/total": 21576276, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 461969 + }, + { + "epoch": 22.34375, + "grad_norm": 0.00017197569832205772, + "learning_rate": 3.066005247943333e-05, + "loss": 1.4886504686728586e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1430, + "tokens/total": 21591356, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 462296 + }, + { + "epoch": 22.359375, + "grad_norm": 0.00011102500138804317, + "learning_rate": 3.059898094696385e-05, + "loss": 1.4149113667372148e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1431, + "tokens/total": 21606484, + "tokens/train_per_sec_per_gpu": 10.28, + "tokens/trainable": 462627 + }, + { + "epoch": 22.375, + "grad_norm": 4.020753476652317e-05, + "learning_rate": 3.053797300934379e-05, + "loss": 1.1528973118402064e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1432, + "tokens/total": 21621664, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 462954 + }, + { + "epoch": 22.390625, + "grad_norm": 2.0071884136996232e-05, + "learning_rate": 3.0477028825574316e-05, + "loss": 6.989898793108296e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1433, + "tokens/total": 21636692, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 463276 + }, + { + "epoch": 22.40625, + "grad_norm": 8.944695582613349e-05, + "learning_rate": 3.041614855449042e-05, + "loss": 1.5249954685714329e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1434, + "tokens/total": 21651872, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 463600 + }, + { + "epoch": 22.421875, + "grad_norm": 5.098918336443603e-05, + "learning_rate": 3.035533235476055e-05, + "loss": 1.1298203617116087e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1435, + "tokens/total": 21666928, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 463925 + }, + { + "epoch": 22.4375, + "grad_norm": 4.565237395581789e-05, + "learning_rate": 3.029458038488618e-05, + "loss": 1.3225826478446834e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1436, + "tokens/total": 21682028, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 464252 + }, + { + "epoch": 22.453125, + "grad_norm": 8.736179006518796e-05, + "learning_rate": 3.0233892803201357e-05, + "loss": 1.7099266642617295e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1437, + "tokens/total": 21697100, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 464575 + }, + { + "epoch": 22.46875, + "grad_norm": 2.7530648367246613e-05, + "learning_rate": 3.017326976787233e-05, + "loss": 9.483637768425979e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1438, + "tokens/total": 21712152, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 464899 + }, + { + "epoch": 22.484375, + "grad_norm": 3.4788834454957396e-05, + "learning_rate": 3.011271143689711e-05, + "loss": 1.1656396736725583e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1439, + "tokens/total": 21727268, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 465215 + }, + { + "epoch": 22.5, + "grad_norm": 2.744857374636922e-05, + "learning_rate": 3.005221796810513e-05, + "loss": 9.510097811471496e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1440, + "tokens/total": 21742364, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 465535 + }, + { + "epoch": 22.515625, + "grad_norm": 3.3228836400667205e-05, + "learning_rate": 2.999178951915666e-05, + "loss": 1.0062655064757564e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1441, + "tokens/total": 21757360, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 465859 + }, + { + "epoch": 22.53125, + "grad_norm": 2.9666920454474166e-05, + "learning_rate": 2.993142624754265e-05, + "loss": 7.012167202447017e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1442, + "tokens/total": 21772412, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 466183 + }, + { + "epoch": 22.546875, + "grad_norm": 7.28718878235668e-05, + "learning_rate": 2.9871128310584094e-05, + "loss": 1.7174085087390267e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1443, + "tokens/total": 21787420, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 466504 + }, + { + "epoch": 22.5625, + "grad_norm": 3.14424469252117e-05, + "learning_rate": 2.9810895865431732e-05, + "loss": 9.947777925845003e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1444, + "tokens/total": 21802536, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 466828 + }, + { + "epoch": 22.578125, + "grad_norm": 2.9212955269031227e-05, + "learning_rate": 2.975072906906563e-05, + "loss": 1.057870008480677e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1445, + "tokens/total": 21817480, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 467147 + }, + { + "epoch": 22.59375, + "grad_norm": 2.700121513043996e-05, + "learning_rate": 2.9690628078294714e-05, + "loss": 8.941700002651487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1446, + "tokens/total": 21832616, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 467470 + }, + { + "epoch": 22.609375, + "grad_norm": 2.5088831534958445e-05, + "learning_rate": 2.9630593049756472e-05, + "loss": 7.274447852978483e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1447, + "tokens/total": 21847652, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 467793 + }, + { + "epoch": 22.625, + "grad_norm": 3.537940574460663e-05, + "learning_rate": 2.957062413991643e-05, + "loss": 1.0811711490532616e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1448, + "tokens/total": 21862660, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 468113 + }, + { + "epoch": 22.640625, + "grad_norm": 2.9958478990010917e-05, + "learning_rate": 2.9510721505067806e-05, + "loss": 1.0096003961734823e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1449, + "tokens/total": 21877712, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 468434 + }, + { + "epoch": 22.65625, + "grad_norm": 3.606174504966475e-05, + "learning_rate": 2.9450885301331065e-05, + "loss": 1.1756588946809643e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1450, + "tokens/total": 21892844, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 468761 + }, + { + "epoch": 22.671875, + "grad_norm": 3.0027202228666283e-05, + "learning_rate": 2.9391115684653612e-05, + "loss": 9.80105596681824e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1451, + "tokens/total": 21908044, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 469089 + }, + { + "epoch": 22.6875, + "grad_norm": 2.73227778961882e-05, + "learning_rate": 2.9331412810809232e-05, + "loss": 1.0051319350168342e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1452, + "tokens/total": 21923252, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 469410 + }, + { + "epoch": 22.703125, + "grad_norm": 2.783164927677717e-05, + "learning_rate": 2.9271776835397785e-05, + "loss": 9.353128689326695e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1453, + "tokens/total": 21938288, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 469737 + }, + { + "epoch": 22.71875, + "grad_norm": 3.60897793143522e-05, + "learning_rate": 2.9212207913844775e-05, + "loss": 1.218863758367661e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1454, + "tokens/total": 21953392, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 470060 + }, + { + "epoch": 22.734375, + "grad_norm": 3.507649671519175e-05, + "learning_rate": 2.915270620140099e-05, + "loss": 1.1126799108751584e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1455, + "tokens/total": 21968500, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 470381 + }, + { + "epoch": 22.75, + "grad_norm": 3.535125870257616e-05, + "learning_rate": 2.909327185314199e-05, + "loss": 1.0780609045468736e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1456, + "tokens/total": 21983616, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 470706 + }, + { + "epoch": 22.765625, + "grad_norm": 3.5386954550631344e-05, + "learning_rate": 2.903390502396781e-05, + "loss": 1.120894239647896e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1457, + "tokens/total": 21998636, + "tokens/train_per_sec_per_gpu": 8.69, + "tokens/trainable": 471026 + }, + { + "epoch": 22.78125, + "grad_norm": 2.547773146943655e-05, + "learning_rate": 2.8974605868602488e-05, + "loss": 7.815493745511048e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1458, + "tokens/total": 22013748, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 471347 + }, + { + "epoch": 22.796875, + "grad_norm": 6.090974056860432e-05, + "learning_rate": 2.8915374541593737e-05, + "loss": 1.364067429676652e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1459, + "tokens/total": 22028824, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 471669 + }, + { + "epoch": 22.8125, + "grad_norm": 6.108806701377034e-05, + "learning_rate": 2.8856211197312432e-05, + "loss": 1.4375473256222904e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1460, + "tokens/total": 22044072, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 471993 + }, + { + "epoch": 22.828125, + "grad_norm": 2.0681460227933712e-05, + "learning_rate": 2.8797115989952305e-05, + "loss": 7.486701747438929e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1461, + "tokens/total": 22059212, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 472310 + }, + { + "epoch": 22.84375, + "grad_norm": 5.616158523480408e-05, + "learning_rate": 2.873808907352949e-05, + "loss": 1.4298193491413258e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1462, + "tokens/total": 22074312, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 472635 + }, + { + "epoch": 22.859375, + "grad_norm": 3.480156738078222e-05, + "learning_rate": 2.8679130601882144e-05, + "loss": 1.221643174176279e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1463, + "tokens/total": 22089440, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 472956 + }, + { + "epoch": 22.875, + "grad_norm": 2.6673551474232227e-05, + "learning_rate": 2.862024072867007e-05, + "loss": 9.592342848918634e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1464, + "tokens/total": 22104464, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 473282 + }, + { + "epoch": 22.890625, + "grad_norm": 2.5087672838708386e-05, + "learning_rate": 2.8561419607374214e-05, + "loss": 8.890181106835371e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1465, + "tokens/total": 22119676, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 473608 + }, + { + "epoch": 22.90625, + "grad_norm": 2.1075858967378736e-05, + "learning_rate": 2.8502667391296435e-05, + "loss": 7.025963100204535e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1466, + "tokens/total": 22134644, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 473930 + }, + { + "epoch": 22.921875, + "grad_norm": 4.733083915198222e-05, + "learning_rate": 2.8443984233558907e-05, + "loss": 1.0771437928269734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1467, + "tokens/total": 22149764, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 474252 + }, + { + "epoch": 22.9375, + "grad_norm": 2.6353798602940515e-05, + "learning_rate": 2.838537028710394e-05, + "loss": 9.020517381941318e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1468, + "tokens/total": 22164880, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 474579 + }, + { + "epoch": 22.953125, + "grad_norm": 7.327018101932481e-05, + "learning_rate": 2.8326825704693316e-05, + "loss": 1.5085292943695094e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1469, + "tokens/total": 22180092, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 474902 + }, + { + "epoch": 22.96875, + "grad_norm": 2.935310658358503e-05, + "learning_rate": 2.8268350638908182e-05, + "loss": 8.673461024955031e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1470, + "tokens/total": 22195132, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 475225 + }, + { + "epoch": 22.984375, + "grad_norm": 2.9737793738604523e-05, + "learning_rate": 2.8209945242148395e-05, + "loss": 9.26823247482389e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1471, + "tokens/total": 22210088, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 475547 + }, + { + "epoch": 23.0, + "grad_norm": 3.8746547943446785e-05, + "learning_rate": 2.815160966663235e-05, + "loss": 1.2070670436514774e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1472, + "tokens/total": 22225280, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 475870 + }, + { + "epoch": 23.015625, + "grad_norm": 3.5526441934052855e-05, + "learning_rate": 2.809334406439633e-05, + "loss": 1.2495027021941496e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1473, + "tokens/total": 22240364, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 476193 + }, + { + "epoch": 23.03125, + "grad_norm": 2.8916938390466385e-05, + "learning_rate": 2.8035148587294387e-05, + "loss": 9.873755288936081e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1474, + "tokens/total": 22255444, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 476515 + }, + { + "epoch": 23.046875, + "grad_norm": 2.5116296455962583e-05, + "learning_rate": 2.7977023386997735e-05, + "loss": 7.543357583017496e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1475, + "tokens/total": 22270416, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 476841 + }, + { + "epoch": 23.0625, + "grad_norm": 1.987470022868365e-05, + "learning_rate": 2.7918968614994445e-05, + "loss": 6.735944566571561e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1476, + "tokens/total": 22285748, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 477164 + }, + { + "epoch": 23.078125, + "grad_norm": 3.0204821086954325e-05, + "learning_rate": 2.7860984422589016e-05, + "loss": 9.641289580031298e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1477, + "tokens/total": 22300808, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 477490 + }, + { + "epoch": 23.09375, + "grad_norm": 7.985880802152678e-05, + "learning_rate": 2.7803070960902062e-05, + "loss": 1.604402768862201e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1478, + "tokens/total": 22315984, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 477814 + }, + { + "epoch": 23.109375, + "grad_norm": 2.991511246364098e-05, + "learning_rate": 2.7745228380869798e-05, + "loss": 1.0028306860476732e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1479, + "tokens/total": 22331044, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 478136 + }, + { + "epoch": 23.125, + "grad_norm": 2.8012733309878968e-05, + "learning_rate": 2.768745683324372e-05, + "loss": 8.814103011900443e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1480, + "tokens/total": 22346100, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 478457 + }, + { + "epoch": 23.140625, + "grad_norm": 2.2160233129397966e-05, + "learning_rate": 2.762975646859019e-05, + "loss": 8.078620226115163e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1481, + "tokens/total": 22361264, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 478778 + }, + { + "epoch": 23.15625, + "grad_norm": 3.799797923420556e-05, + "learning_rate": 2.7572127437290053e-05, + "loss": 1.1203700296391617e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1482, + "tokens/total": 22376396, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 479103 + }, + { + "epoch": 23.171875, + "grad_norm": 3.132782148895785e-05, + "learning_rate": 2.7514569889538278e-05, + "loss": 1.0528925713515491e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1483, + "tokens/total": 22391520, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 479426 + }, + { + "epoch": 23.1875, + "grad_norm": 2.666118962224573e-05, + "learning_rate": 2.745708397534348e-05, + "loss": 9.056093972503731e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1484, + "tokens/total": 22406480, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 479748 + }, + { + "epoch": 23.203125, + "grad_norm": 4.181926851742901e-05, + "learning_rate": 2.7399669844527613e-05, + "loss": 1.3681499240192352e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1485, + "tokens/total": 22421664, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 480073 + }, + { + "epoch": 23.21875, + "grad_norm": 2.8672833650489338e-05, + "learning_rate": 2.7342327646725506e-05, + "loss": 8.50804667607008e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1486, + "tokens/total": 22436748, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 480396 + }, + { + "epoch": 23.234375, + "grad_norm": 2.036990554188378e-05, + "learning_rate": 2.7285057531384595e-05, + "loss": 7.639360433131515e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1487, + "tokens/total": 22451728, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 480720 + }, + { + "epoch": 23.25, + "grad_norm": 2.8006648790324107e-05, + "learning_rate": 2.722785964776433e-05, + "loss": 9.61461296355992e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1488, + "tokens/total": 22466760, + "tokens/train_per_sec_per_gpu": 10.13, + "tokens/trainable": 481042 + }, + { + "epoch": 23.265625, + "grad_norm": 2.8975056920899078e-05, + "learning_rate": 2.7170734144936028e-05, + "loss": 1.0470710094523383e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1489, + "tokens/total": 22481856, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 481368 + }, + { + "epoch": 23.28125, + "grad_norm": 2.5139665012829937e-05, + "learning_rate": 2.711368117178228e-05, + "loss": 8.53116091548145e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1490, + "tokens/total": 22497004, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 481696 + }, + { + "epoch": 23.296875, + "grad_norm": 7.887971150921658e-05, + "learning_rate": 2.7056700876996732e-05, + "loss": 1.329665792582091e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1491, + "tokens/total": 22512116, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 482018 + }, + { + "epoch": 23.3125, + "grad_norm": 1.8393631762592122e-05, + "learning_rate": 2.6999793409083496e-05, + "loss": 6.452157776948297e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1492, + "tokens/total": 22527256, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 482344 + }, + { + "epoch": 23.328125, + "grad_norm": 4.02365876652766e-05, + "learning_rate": 2.6942958916356998e-05, + "loss": 9.786932650968083e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1493, + "tokens/total": 22542476, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 482670 + }, + { + "epoch": 23.34375, + "grad_norm": 4.795138738700189e-05, + "learning_rate": 2.6886197546941383e-05, + "loss": 1.1048532542190515e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1494, + "tokens/total": 22557568, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 482987 + }, + { + "epoch": 23.359375, + "grad_norm": 4.525393160292879e-05, + "learning_rate": 2.6829509448770296e-05, + "loss": 1.2275785366000491e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1495, + "tokens/total": 22572848, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 483312 + }, + { + "epoch": 23.375, + "grad_norm": 2.5106150133069605e-05, + "learning_rate": 2.6772894769586332e-05, + "loss": 7.973192168719834e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 1496, + "tokens/total": 22588016, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 483638 + }, + { + "epoch": 23.390625, + "grad_norm": 5.750935088144615e-05, + "learning_rate": 2.671635365694083e-05, + "loss": 1.1274078133283183e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1497, + "tokens/total": 22603080, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 483963 + }, + { + "epoch": 23.40625, + "grad_norm": 3.599207047955133e-05, + "learning_rate": 2.665988625819334e-05, + "loss": 1.1609382681854186e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1498, + "tokens/total": 22618280, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 484287 + }, + { + "epoch": 23.421875, + "grad_norm": 2.4953355023171753e-05, + "learning_rate": 2.6603492720511312e-05, + "loss": 8.579685868426168e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1499, + "tokens/total": 22633228, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 484612 + }, + { + "epoch": 23.4375, + "grad_norm": 6.365439912769943e-05, + "learning_rate": 2.654717319086969e-05, + "loss": 1.6794792827568017e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1500, + "tokens/total": 22648180, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 484938 + }, + { + "epoch": 23.453125, + "grad_norm": 4.2809173464775085e-05, + "learning_rate": 2.6490927816050526e-05, + "loss": 1.1817329550467548e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1501, + "tokens/total": 22663176, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 485259 + }, + { + "epoch": 23.46875, + "grad_norm": 6.405643944162875e-05, + "learning_rate": 2.643475674264265e-05, + "loss": 1.5119351246539736e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1502, + "tokens/total": 22678328, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 485585 + }, + { + "epoch": 23.484375, + "grad_norm": 3.483953696559183e-05, + "learning_rate": 2.637866011704121e-05, + "loss": 8.795482813184208e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1503, + "tokens/total": 22693424, + "tokens/train_per_sec_per_gpu": 10.3, + "tokens/trainable": 485913 + }, + { + "epoch": 23.5, + "grad_norm": 2.58744530583499e-05, + "learning_rate": 2.6322638085447334e-05, + "loss": 8.106429731924436e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1504, + "tokens/total": 22708616, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 486235 + }, + { + "epoch": 23.515625, + "grad_norm": 2.5191327949869446e-05, + "learning_rate": 2.626669079386771e-05, + "loss": 8.908104405236372e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1505, + "tokens/total": 22723648, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 486557 + }, + { + "epoch": 23.53125, + "grad_norm": 2.4037977709667757e-05, + "learning_rate": 2.62108183881143e-05, + "loss": 8.89647481017164e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1506, + "tokens/total": 22738744, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 486879 + }, + { + "epoch": 23.546875, + "grad_norm": 2.3540405891253613e-05, + "learning_rate": 2.6155021013803842e-05, + "loss": 8.386027730011847e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1507, + "tokens/total": 22753912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 487203 + }, + { + "epoch": 23.5625, + "grad_norm": 6.840170681243762e-05, + "learning_rate": 2.6099298816357555e-05, + "loss": 1.0414065627628588e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1508, + "tokens/total": 22769040, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 487527 + }, + { + "epoch": 23.578125, + "grad_norm": 5.226628854870796e-05, + "learning_rate": 2.604365194100068e-05, + "loss": 1.1548304428288247e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1509, + "tokens/total": 22784012, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 487849 + }, + { + "epoch": 23.59375, + "grad_norm": 3.086002834606916e-05, + "learning_rate": 2.5988080532762248e-05, + "loss": 8.696399049767933e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1510, + "tokens/total": 22799172, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 488172 + }, + { + "epoch": 23.609375, + "grad_norm": 3.5357032174943015e-05, + "learning_rate": 2.593258473647448e-05, + "loss": 8.087192782113561e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1511, + "tokens/total": 22814124, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 488495 + }, + { + "epoch": 23.625, + "grad_norm": 6.179608317324892e-05, + "learning_rate": 2.5877164696772638e-05, + "loss": 1.1847064342873637e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1512, + "tokens/total": 22829148, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 488817 + }, + { + "epoch": 23.640625, + "grad_norm": 6.291170575423166e-05, + "learning_rate": 2.5821820558094483e-05, + "loss": 1.177870785795676e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1513, + "tokens/total": 22844272, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 489141 + }, + { + "epoch": 23.65625, + "grad_norm": 8.55255129863508e-05, + "learning_rate": 2.5766552464680015e-05, + "loss": 1.4671383041786612e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1514, + "tokens/total": 22859352, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 489465 + }, + { + "epoch": 23.671875, + "grad_norm": 1.8059470676234923e-05, + "learning_rate": 2.571136056057096e-05, + "loss": 6.182417564559728e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1515, + "tokens/total": 22874376, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 489789 + }, + { + "epoch": 23.6875, + "grad_norm": 2.207538091170136e-05, + "learning_rate": 2.5656244989610568e-05, + "loss": 6.908345540068694e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1516, + "tokens/total": 22889344, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 490112 + }, + { + "epoch": 23.703125, + "grad_norm": 2.9781896955682896e-05, + "learning_rate": 2.560120589544308e-05, + "loss": 9.381828363075329e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1517, + "tokens/total": 22904408, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 490437 + }, + { + "epoch": 23.71875, + "grad_norm": 2.22787512029754e-05, + "learning_rate": 2.5546243421513444e-05, + "loss": 7.067797014315147e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1518, + "tokens/total": 22919460, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 490756 + }, + { + "epoch": 23.734375, + "grad_norm": 2.9510993044823408e-05, + "learning_rate": 2.549135771106693e-05, + "loss": 8.142721981130308e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1519, + "tokens/total": 22934492, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 491085 + }, + { + "epoch": 23.75, + "grad_norm": 3.018093775608577e-05, + "learning_rate": 2.5436548907148706e-05, + "loss": 1.0326223218726227e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1520, + "tokens/total": 22949552, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 491406 + }, + { + "epoch": 23.765625, + "grad_norm": 2.7528696591616608e-05, + "learning_rate": 2.538181715260356e-05, + "loss": 9.374699061481806e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1521, + "tokens/total": 22964652, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 491728 + }, + { + "epoch": 23.78125, + "grad_norm": 3.0921848519938067e-05, + "learning_rate": 2.5327162590075426e-05, + "loss": 9.741996791490237e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1522, + "tokens/total": 22979728, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 492051 + }, + { + "epoch": 23.796875, + "grad_norm": 3.1102154025575146e-05, + "learning_rate": 2.5272585362007067e-05, + "loss": 1.0681022786229732e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1523, + "tokens/total": 22994732, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 492372 + }, + { + "epoch": 23.8125, + "grad_norm": 4.405981962918304e-05, + "learning_rate": 2.5218085610639696e-05, + "loss": 1.1352175306456047e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1524, + "tokens/total": 23009844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 492696 + }, + { + "epoch": 23.828125, + "grad_norm": 0.0001750801020534709, + "learning_rate": 2.5163663478012618e-05, + "loss": 2.752464979494107e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1525, + "tokens/total": 23025024, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 493021 + }, + { + "epoch": 23.84375, + "grad_norm": 5.994981256662868e-05, + "learning_rate": 2.5109319105962835e-05, + "loss": 1.335818865300098e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1526, + "tokens/total": 23040172, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 493343 + }, + { + "epoch": 23.859375, + "grad_norm": 2.7728914574254304e-05, + "learning_rate": 2.5055052636124666e-05, + "loss": 9.812181360757677e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1527, + "tokens/total": 23055144, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 493666 + }, + { + "epoch": 23.875, + "grad_norm": 3.0788203730480745e-05, + "learning_rate": 2.500086420992942e-05, + "loss": 9.784633903109352e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1528, + "tokens/total": 23070124, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 493987 + }, + { + "epoch": 23.890625, + "grad_norm": 3.504412961774506e-05, + "learning_rate": 2.4946753968605012e-05, + "loss": 9.910314702210599e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1529, + "tokens/total": 23085176, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 494304 + }, + { + "epoch": 23.90625, + "grad_norm": 2.3705826606601477e-05, + "learning_rate": 2.4892722053175583e-05, + "loss": 8.391593269152509e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1530, + "tokens/total": 23100172, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 494624 + }, + { + "epoch": 23.921875, + "grad_norm": 2.2259455363382585e-05, + "learning_rate": 2.4838768604461125e-05, + "loss": 7.748081998215639e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1531, + "tokens/total": 23115288, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 494946 + }, + { + "epoch": 23.9375, + "grad_norm": 2.7372265321901068e-05, + "learning_rate": 2.4784893763077116e-05, + "loss": 9.616346687835176e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1532, + "tokens/total": 23130428, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 495269 + }, + { + "epoch": 23.953125, + "grad_norm": 2.369913454458583e-05, + "learning_rate": 2.473109766943422e-05, + "loss": 8.517500305060821e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1533, + "tokens/total": 23145600, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 495592 + }, + { + "epoch": 23.96875, + "grad_norm": 1.759544829837978e-05, + "learning_rate": 2.467738046373782e-05, + "loss": 6.209381240296352e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1534, + "tokens/total": 23160692, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 495915 + }, + { + "epoch": 23.984375, + "grad_norm": 8.024468115763739e-05, + "learning_rate": 2.4623742285987702e-05, + "loss": 1.6640000239931396e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1535, + "tokens/total": 23175800, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 496239 + }, + { + "epoch": 24.0, + "grad_norm": 3.559091055649333e-05, + "learning_rate": 2.4570183275977694e-05, + "loss": 1.0301279189661727e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1536, + "tokens/total": 23190820, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 496560 + }, + { + "epoch": 24.015625, + "grad_norm": 3.571200795704499e-05, + "learning_rate": 2.4516703573295287e-05, + "loss": 1.1727769333447213e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1537, + "tokens/total": 23205920, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 496882 + }, + { + "epoch": 24.03125, + "grad_norm": 2.6527348381932825e-05, + "learning_rate": 2.4463303317321305e-05, + "loss": 9.770362794370158e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1538, + "tokens/total": 23220964, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 497207 + }, + { + "epoch": 24.046875, + "grad_norm": 1.904603595903609e-05, + "learning_rate": 2.4409982647229465e-05, + "loss": 6.607840532524278e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1539, + "tokens/total": 23236064, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 497538 + }, + { + "epoch": 24.0625, + "grad_norm": 2.945854066638276e-05, + "learning_rate": 2.4356741701986123e-05, + "loss": 9.6028679763549e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1540, + "tokens/total": 23251248, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 497859 + }, + { + "epoch": 24.078125, + "grad_norm": 3.137306703138165e-05, + "learning_rate": 2.4303580620349787e-05, + "loss": 1.0187951602347312e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1541, + "tokens/total": 23266332, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 498180 + }, + { + "epoch": 24.09375, + "grad_norm": 2.409764420008287e-05, + "learning_rate": 2.4250499540870918e-05, + "loss": 8.355857517017284e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1542, + "tokens/total": 23281472, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 498497 + }, + { + "epoch": 24.109375, + "grad_norm": 6.84373953845352e-05, + "learning_rate": 2.4197498601891344e-05, + "loss": 1.4000895589560969e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1543, + "tokens/total": 23296536, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 498822 + }, + { + "epoch": 24.125, + "grad_norm": 1.6333911844412796e-05, + "learning_rate": 2.4144577941544145e-05, + "loss": 5.686589474862558e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1544, + "tokens/total": 23311696, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 499148 + }, + { + "epoch": 24.140625, + "grad_norm": 4.281090514268726e-05, + "learning_rate": 2.4091737697753107e-05, + "loss": 1.0731685051723616e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1545, + "tokens/total": 23326708, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 499474 + }, + { + "epoch": 24.15625, + "grad_norm": 2.6035031623905525e-05, + "learning_rate": 2.403897800823246e-05, + "loss": 7.329423397095525e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1546, + "tokens/total": 23341868, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 499797 + }, + { + "epoch": 24.171875, + "grad_norm": 2.0183882952551357e-05, + "learning_rate": 2.3986299010486463e-05, + "loss": 7.062861300255463e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1547, + "tokens/total": 23357088, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 500125 + }, + { + "epoch": 24.1875, + "grad_norm": 9.018573473440483e-05, + "learning_rate": 2.393370084180912e-05, + "loss": 2.1344849301385693e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1548, + "tokens/total": 23372184, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 500451 + }, + { + "epoch": 24.203125, + "grad_norm": 3.9043450669851154e-05, + "learning_rate": 2.3881183639283732e-05, + "loss": 1.0775781902339077e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1549, + "tokens/total": 23387316, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 500769 + }, + { + "epoch": 24.21875, + "grad_norm": 4.456886381376535e-05, + "learning_rate": 2.3828747539782608e-05, + "loss": 8.276547873720119e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1550, + "tokens/total": 23402500, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 501086 + }, + { + "epoch": 24.234375, + "grad_norm": 1.8638902474776842e-05, + "learning_rate": 2.3776392679966642e-05, + "loss": 6.636234957113629e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1551, + "tokens/total": 23417520, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 501407 + }, + { + "epoch": 24.25, + "grad_norm": 7.891295535955578e-05, + "learning_rate": 2.3724119196285073e-05, + "loss": 1.4655377071903786e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1552, + "tokens/total": 23432652, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 501731 + }, + { + "epoch": 24.265625, + "grad_norm": 3.415428727748804e-05, + "learning_rate": 2.367192722497499e-05, + "loss": 1.0131022918358212e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1553, + "tokens/total": 23447672, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 502052 + }, + { + "epoch": 24.28125, + "grad_norm": 3.800429476541467e-05, + "learning_rate": 2.361981690206107e-05, + "loss": 1.172641077573644e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1554, + "tokens/total": 23462664, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 502377 + }, + { + "epoch": 24.296875, + "grad_norm": 3.729139280039817e-05, + "learning_rate": 2.356778836335518e-05, + "loss": 1.1271088169451104e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1555, + "tokens/total": 23477764, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 502705 + }, + { + "epoch": 24.3125, + "grad_norm": 2.1563466361840256e-05, + "learning_rate": 2.3515841744456036e-05, + "loss": 7.595077704536379e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1556, + "tokens/total": 23492880, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 503027 + }, + { + "epoch": 24.328125, + "grad_norm": 3.536889562383294e-05, + "learning_rate": 2.3463977180748894e-05, + "loss": 1.0899613016590592e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1557, + "tokens/total": 23507924, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 503353 + }, + { + "epoch": 24.34375, + "grad_norm": 6.31110742688179e-05, + "learning_rate": 2.341219480740508e-05, + "loss": 8.329610068358306e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1558, + "tokens/total": 23522900, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 503672 + }, + { + "epoch": 24.359375, + "grad_norm": 2.2952717699809e-05, + "learning_rate": 2.3360494759381786e-05, + "loss": 7.459504445250786e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1559, + "tokens/total": 23537884, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 504002 + }, + { + "epoch": 24.375, + "grad_norm": 2.34188210015418e-05, + "learning_rate": 2.330887717142159e-05, + "loss": 8.353144949069247e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1560, + "tokens/total": 23552944, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 504328 + }, + { + "epoch": 24.390625, + "grad_norm": 3.525171268847771e-05, + "learning_rate": 2.3257342178052214e-05, + "loss": 9.122993560595205e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1561, + "tokens/total": 23568188, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 504653 + }, + { + "epoch": 24.40625, + "grad_norm": 1.9179042283212766e-05, + "learning_rate": 2.3205889913586044e-05, + "loss": 6.634973601649108e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1562, + "tokens/total": 23583404, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 504980 + }, + { + "epoch": 24.421875, + "grad_norm": 2.6049543521367013e-05, + "learning_rate": 2.3154520512119936e-05, + "loss": 8.618727633802337e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1563, + "tokens/total": 23598636, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 505300 + }, + { + "epoch": 24.4375, + "grad_norm": 2.4500222934875637e-05, + "learning_rate": 2.3103234107534718e-05, + "loss": 8.121725727505691e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1564, + "tokens/total": 23613708, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 505627 + }, + { + "epoch": 24.453125, + "grad_norm": 2.5563824237906374e-05, + "learning_rate": 2.3052030833494976e-05, + "loss": 8.494625944877043e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1565, + "tokens/total": 23628916, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 505952 + }, + { + "epoch": 24.46875, + "grad_norm": 3.599560659495182e-05, + "learning_rate": 2.3000910823448554e-05, + "loss": 1.0245964858768275e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1566, + "tokens/total": 23643948, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 506271 + }, + { + "epoch": 24.484375, + "grad_norm": 2.245177711301949e-05, + "learning_rate": 2.294987421062637e-05, + "loss": 7.594973681079864e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1567, + "tokens/total": 23659212, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 506598 + }, + { + "epoch": 24.5, + "grad_norm": 0.00024060698342509568, + "learning_rate": 2.2898921128041928e-05, + "loss": 7.086685513968405e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1568, + "tokens/total": 23674332, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 506922 + }, + { + "epoch": 24.515625, + "grad_norm": 2.9976547011756338e-05, + "learning_rate": 2.2848051708491098e-05, + "loss": 1.0558659369053203e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1569, + "tokens/total": 23689316, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 507246 + }, + { + "epoch": 24.53125, + "grad_norm": 2.6876921765506268e-05, + "learning_rate": 2.279726608455162e-05, + "loss": 8.869580483406025e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1570, + "tokens/total": 23704384, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 507566 + }, + { + "epoch": 24.546875, + "grad_norm": 3.0953684472478926e-05, + "learning_rate": 2.2746564388582926e-05, + "loss": 1.0285805274179438e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1571, + "tokens/total": 23719504, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 507890 + }, + { + "epoch": 24.5625, + "grad_norm": 3.0376068025361747e-05, + "learning_rate": 2.2695946752725654e-05, + "loss": 8.761901426623808e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1572, + "tokens/total": 23734576, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 508213 + }, + { + "epoch": 24.578125, + "grad_norm": 2.7791536922450177e-05, + "learning_rate": 2.2645413308901387e-05, + "loss": 9.396317750542948e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1573, + "tokens/total": 23749596, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 508529 + }, + { + "epoch": 24.59375, + "grad_norm": 1.636040542507544e-05, + "learning_rate": 2.259496418881228e-05, + "loss": 5.743985411754693e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1574, + "tokens/total": 23764704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 508852 + }, + { + "epoch": 24.609375, + "grad_norm": 3.18798738589976e-05, + "learning_rate": 2.2544599523940698e-05, + "loss": 1.1175058034496033e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1575, + "tokens/total": 23779744, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 509176 + }, + { + "epoch": 24.625, + "grad_norm": 2.74404574156506e-05, + "learning_rate": 2.249431944554895e-05, + "loss": 9.601521924196277e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1576, + "tokens/total": 23794880, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 509501 + }, + { + "epoch": 24.640625, + "grad_norm": 8.264268399216235e-05, + "learning_rate": 2.2444124084678846e-05, + "loss": 1.2617960010175011e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1577, + "tokens/total": 23809888, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 509832 + }, + { + "epoch": 24.65625, + "grad_norm": 1.9746436009882018e-05, + "learning_rate": 2.239401357215141e-05, + "loss": 6.211459435689903e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1578, + "tokens/total": 23825052, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 510161 + }, + { + "epoch": 24.671875, + "grad_norm": 5.7905672292690724e-05, + "learning_rate": 2.2343988038566527e-05, + "loss": 1.2012189927190775e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1579, + "tokens/total": 23840176, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 510487 + }, + { + "epoch": 24.6875, + "grad_norm": 2.2979529603617266e-05, + "learning_rate": 2.2294047614302638e-05, + "loss": 7.203051382020931e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1580, + "tokens/total": 23855280, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 510808 + }, + { + "epoch": 24.703125, + "grad_norm": 4.9219652282772586e-05, + "learning_rate": 2.224419242951633e-05, + "loss": 1.2195703220641008e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1581, + "tokens/total": 23870400, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 511126 + }, + { + "epoch": 24.71875, + "grad_norm": 2.1632806237903424e-05, + "learning_rate": 2.2194422614142064e-05, + "loss": 7.551756766588369e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1582, + "tokens/total": 23885624, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 511447 + }, + { + "epoch": 24.734375, + "grad_norm": 2.752237742242869e-05, + "learning_rate": 2.214473829789177e-05, + "loss": 8.773413355811499e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1583, + "tokens/total": 23900688, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 511768 + }, + { + "epoch": 24.75, + "grad_norm": 3.673569153761491e-05, + "learning_rate": 2.2095139610254622e-05, + "loss": 9.840267694016802e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1584, + "tokens/total": 23915852, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 512090 + }, + { + "epoch": 24.765625, + "grad_norm": 1.973571124835871e-05, + "learning_rate": 2.204562668049652e-05, + "loss": 6.495125717265182e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1585, + "tokens/total": 23930944, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 512414 + }, + { + "epoch": 24.78125, + "grad_norm": 0.00016510198474861681, + "learning_rate": 2.1996199637659957e-05, + "loss": 2.753580702119507e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1586, + "tokens/total": 23946024, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 512732 + }, + { + "epoch": 24.796875, + "grad_norm": 2.4856959498720244e-05, + "learning_rate": 2.194685861056351e-05, + "loss": 6.061839030735428e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1587, + "tokens/total": 23960988, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 513054 + }, + { + "epoch": 24.8125, + "grad_norm": 2.18275254155742e-05, + "learning_rate": 2.1897603727801656e-05, + "loss": 7.624889803992119e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1588, + "tokens/total": 23975988, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 513379 + }, + { + "epoch": 24.828125, + "grad_norm": 3.218476558686234e-05, + "learning_rate": 2.1848435117744266e-05, + "loss": 9.836898016146733e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1589, + "tokens/total": 23991136, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 513703 + }, + { + "epoch": 24.84375, + "grad_norm": 2.7437352400738746e-05, + "learning_rate": 2.1799352908536442e-05, + "loss": 8.261332595793647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1590, + "tokens/total": 24006212, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 514023 + }, + { + "epoch": 24.859375, + "grad_norm": 5.748223338741809e-05, + "learning_rate": 2.175035722809806e-05, + "loss": 1.2285840966796968e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1591, + "tokens/total": 24021360, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 514345 + }, + { + "epoch": 24.875, + "grad_norm": 3.5715402191272005e-05, + "learning_rate": 2.1701448204123515e-05, + "loss": 9.862677643468487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1592, + "tokens/total": 24036532, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 514665 + }, + { + "epoch": 24.890625, + "grad_norm": 2.1580019165412523e-05, + "learning_rate": 2.165262596408132e-05, + "loss": 6.471384494943777e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1593, + "tokens/total": 24051756, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 514991 + }, + { + "epoch": 24.90625, + "grad_norm": 2.1927951820543967e-05, + "learning_rate": 2.160389063521382e-05, + "loss": 7.588346306874882e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1594, + "tokens/total": 24066808, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 515317 + }, + { + "epoch": 24.921875, + "grad_norm": 2.6974768843501806e-05, + "learning_rate": 2.1555242344536873e-05, + "loss": 9.095548989535018e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1595, + "tokens/total": 24082064, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 515646 + }, + { + "epoch": 24.9375, + "grad_norm": 2.268885145895183e-05, + "learning_rate": 2.150668121883947e-05, + "loss": 7.963754455886374e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1596, + "tokens/total": 24097152, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 515965 + }, + { + "epoch": 24.953125, + "grad_norm": 1.8854412701330148e-05, + "learning_rate": 2.1458207384683428e-05, + "loss": 6.418462135115988e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1597, + "tokens/total": 24112152, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 516286 + }, + { + "epoch": 24.96875, + "grad_norm": 6.889060023240745e-05, + "learning_rate": 2.1409820968403056e-05, + "loss": 1.1790785947596305e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1598, + "tokens/total": 24127136, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 516605 + }, + { + "epoch": 24.984375, + "grad_norm": 4.589032323565334e-05, + "learning_rate": 2.1361522096104875e-05, + "loss": 1.0940000265691197e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1599, + "tokens/total": 24142308, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 516925 + }, + { + "epoch": 25.0, + "grad_norm": 4.5295240852283314e-05, + "learning_rate": 2.1313310893667192e-05, + "loss": 1.1007299463017262e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1600, + "tokens/total": 24157356, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 517250 + }, + { + "epoch": 25.015625, + "grad_norm": 1.6421288819401525e-05, + "learning_rate": 2.126518748673986e-05, + "loss": 5.67049426081212e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1601, + "tokens/total": 24172344, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 517575 + }, + { + "epoch": 25.03125, + "grad_norm": 3.779712278628722e-05, + "learning_rate": 2.1217152000743884e-05, + "loss": 9.268826488550985e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1602, + "tokens/total": 24187484, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 517903 + }, + { + "epoch": 25.046875, + "grad_norm": 2.2371923478203826e-05, + "learning_rate": 2.1169204560871172e-05, + "loss": 7.317535164474975e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1603, + "tokens/total": 24202512, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 518233 + }, + { + "epoch": 25.0625, + "grad_norm": 4.400189936859533e-05, + "learning_rate": 2.1121345292084127e-05, + "loss": 6.888378720759647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1604, + "tokens/total": 24217588, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 518558 + }, + { + "epoch": 25.078125, + "grad_norm": 4.706600884674117e-05, + "learning_rate": 2.1073574319115363e-05, + "loss": 1.1353794207025203e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1605, + "tokens/total": 24232700, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 518881 + }, + { + "epoch": 25.09375, + "grad_norm": 2.0048093574587256e-05, + "learning_rate": 2.102589176646736e-05, + "loss": 7.138766591197054e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1606, + "tokens/total": 24247884, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 519205 + }, + { + "epoch": 25.109375, + "grad_norm": 2.453859087836463e-05, + "learning_rate": 2.0978297758412214e-05, + "loss": 8.777575430940487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1607, + "tokens/total": 24262984, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 519530 + }, + { + "epoch": 25.125, + "grad_norm": 2.0335208319011144e-05, + "learning_rate": 2.0930792418991184e-05, + "loss": 7.199782317002246e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1608, + "tokens/total": 24278048, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 519853 + }, + { + "epoch": 25.140625, + "grad_norm": 1.6501746358699165e-05, + "learning_rate": 2.088337587201447e-05, + "loss": 5.484105258801719e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1609, + "tokens/total": 24293232, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 520178 + }, + { + "epoch": 25.15625, + "grad_norm": 2.0374072846607305e-05, + "learning_rate": 2.083604824106085e-05, + "loss": 7.384285254374845e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1610, + "tokens/total": 24308332, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 520503 + }, + { + "epoch": 25.171875, + "grad_norm": 2.041388870566152e-05, + "learning_rate": 2.078880964947736e-05, + "loss": 7.345047379203606e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1611, + "tokens/total": 24323444, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 520825 + }, + { + "epoch": 25.1875, + "grad_norm": 2.8196463972562924e-05, + "learning_rate": 2.0741660220379012e-05, + "loss": 8.50919434469688e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1612, + "tokens/total": 24338496, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 521149 + }, + { + "epoch": 25.203125, + "grad_norm": 2.948411929537542e-05, + "learning_rate": 2.069460007664838e-05, + "loss": 8.463563290206366e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1613, + "tokens/total": 24353644, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 521469 + }, + { + "epoch": 25.21875, + "grad_norm": 1.9328505004523322e-05, + "learning_rate": 2.0647629340935387e-05, + "loss": 6.96961251378525e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1614, + "tokens/total": 24368700, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 521797 + }, + { + "epoch": 25.234375, + "grad_norm": 2.1240404748823494e-05, + "learning_rate": 2.0600748135656926e-05, + "loss": 7.201428502412455e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1615, + "tokens/total": 24383768, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 522116 + }, + { + "epoch": 25.25, + "grad_norm": 3.0129478545859456e-05, + "learning_rate": 2.055395658299656e-05, + "loss": 9.911999541145633e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1616, + "tokens/total": 24398892, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 522437 + }, + { + "epoch": 25.265625, + "grad_norm": 3.8463720557047054e-05, + "learning_rate": 2.0507254804904158e-05, + "loss": 1.13869964479818e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1617, + "tokens/total": 24413896, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 522756 + }, + { + "epoch": 25.28125, + "grad_norm": 2.24506038648542e-05, + "learning_rate": 2.0460642923095662e-05, + "loss": 7.0230657911452e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1618, + "tokens/total": 24429044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 523075 + }, + { + "epoch": 25.296875, + "grad_norm": 2.013606354012154e-05, + "learning_rate": 2.0414121059052703e-05, + "loss": 7.067246201586386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1619, + "tokens/total": 24444152, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 523399 + }, + { + "epoch": 25.3125, + "grad_norm": 2.2038357201381586e-05, + "learning_rate": 2.03676893340223e-05, + "loss": 8.087484957286506e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1620, + "tokens/total": 24459260, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 523720 + }, + { + "epoch": 25.328125, + "grad_norm": 6.897580897202715e-05, + "learning_rate": 2.0321347869016542e-05, + "loss": 1.4260931493481621e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1621, + "tokens/total": 24474312, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 524049 + }, + { + "epoch": 25.34375, + "grad_norm": 4.169297972111963e-05, + "learning_rate": 2.0275096784812313e-05, + "loss": 8.777150810601597e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1622, + "tokens/total": 24489468, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 524376 + }, + { + "epoch": 25.359375, + "grad_norm": 6.565138755831867e-05, + "learning_rate": 2.022893620195091e-05, + "loss": 1.0072553777717985e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1623, + "tokens/total": 24504508, + "tokens/train_per_sec_per_gpu": 8.96, + "tokens/trainable": 524695 + }, + { + "epoch": 25.375, + "grad_norm": 5.5061398597899824e-05, + "learning_rate": 2.0182866240737768e-05, + "loss": 1.23968425214116e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1624, + "tokens/total": 24519640, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 525026 + }, + { + "epoch": 25.390625, + "grad_norm": 3.167263275827281e-05, + "learning_rate": 2.013688702124214e-05, + "loss": 1.015626594380592e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1625, + "tokens/total": 24534840, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 525348 + }, + { + "epoch": 25.40625, + "grad_norm": 1.805424653866794e-05, + "learning_rate": 2.0090998663296813e-05, + "loss": 6.429015684261685e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1626, + "tokens/total": 24549984, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 525672 + }, + { + "epoch": 25.421875, + "grad_norm": 2.431163920846302e-05, + "learning_rate": 2.0045201286497722e-05, + "loss": 6.546757731484831e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1627, + "tokens/total": 24565016, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 525997 + }, + { + "epoch": 25.4375, + "grad_norm": 3.063264739466831e-05, + "learning_rate": 1.9999495010203705e-05, + "loss": 1.019842329696985e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1628, + "tokens/total": 24580144, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 526320 + }, + { + "epoch": 25.453125, + "grad_norm": 2.9408269256236963e-05, + "learning_rate": 1.9953879953536188e-05, + "loss": 9.183705174109491e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1629, + "tokens/total": 24595228, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 526645 + }, + { + "epoch": 25.46875, + "grad_norm": 5.545400927076116e-05, + "learning_rate": 1.9908356235378808e-05, + "loss": 1.0216929240414174e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1630, + "tokens/total": 24610336, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 526970 + }, + { + "epoch": 25.484375, + "grad_norm": 2.461611438775435e-05, + "learning_rate": 1.9862923974377228e-05, + "loss": 8.236488611146342e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1631, + "tokens/total": 24625392, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 527297 + }, + { + "epoch": 25.5, + "grad_norm": 6.758881499990821e-05, + "learning_rate": 1.981758328893866e-05, + "loss": 1.5421583157149144e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1632, + "tokens/total": 24640360, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 527620 + }, + { + "epoch": 25.515625, + "grad_norm": 3.8353002310032025e-05, + "learning_rate": 1.9772334297231732e-05, + "loss": 9.360549029224785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1633, + "tokens/total": 24655468, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 527942 + }, + { + "epoch": 25.53125, + "grad_norm": 5.42439374839887e-05, + "learning_rate": 1.972717711718604e-05, + "loss": 1.1045229939554702e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1634, + "tokens/total": 24670420, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 528267 + }, + { + "epoch": 25.546875, + "grad_norm": 0.00016510246496181935, + "learning_rate": 1.9682111866491943e-05, + "loss": 2.462938937242143e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1635, + "tokens/total": 24685548, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 528589 + }, + { + "epoch": 25.5625, + "grad_norm": 7.279841520357877e-05, + "learning_rate": 1.963713866260014e-05, + "loss": 1.2993832569918595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1636, + "tokens/total": 24700664, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 528915 + }, + { + "epoch": 25.578125, + "grad_norm": 3.379786721779965e-05, + "learning_rate": 1.9592257622721517e-05, + "loss": 1.0423933645142824e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1637, + "tokens/total": 24715768, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 529239 + }, + { + "epoch": 25.59375, + "grad_norm": 3.62376740667969e-05, + "learning_rate": 1.95474688638267e-05, + "loss": 1.007309492706554e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1638, + "tokens/total": 24730812, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 529559 + }, + { + "epoch": 25.609375, + "grad_norm": 2.1051539079053327e-05, + "learning_rate": 1.9502772502645857e-05, + "loss": 6.476227554230718e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1639, + "tokens/total": 24746024, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 529884 + }, + { + "epoch": 25.625, + "grad_norm": 2.5959810955100693e-05, + "learning_rate": 1.945816865566828e-05, + "loss": 9.022110702971986e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1640, + "tokens/total": 24761172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 530209 + }, + { + "epoch": 25.640625, + "grad_norm": 2.528899312892463e-05, + "learning_rate": 1.9413657439142208e-05, + "loss": 8.393857910959923e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1641, + "tokens/total": 24776264, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 530530 + }, + { + "epoch": 25.65625, + "grad_norm": 1.808511296985671e-05, + "learning_rate": 1.9369238969074445e-05, + "loss": 6.354350148285448e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1642, + "tokens/total": 24791412, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 530853 + }, + { + "epoch": 25.671875, + "grad_norm": 2.124305501638446e-05, + "learning_rate": 1.9324913361230052e-05, + "loss": 6.511561423394596e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1643, + "tokens/total": 24806456, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 531175 + }, + { + "epoch": 25.6875, + "grad_norm": 3.0255354431574233e-05, + "learning_rate": 1.9280680731132082e-05, + "loss": 9.824831295190961e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1644, + "tokens/total": 24821460, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 531498 + }, + { + "epoch": 25.703125, + "grad_norm": 2.0916360881528817e-05, + "learning_rate": 1.923654119406131e-05, + "loss": 7.26212419976946e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1645, + "tokens/total": 24836548, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 531815 + }, + { + "epoch": 25.71875, + "grad_norm": 2.139650132448878e-05, + "learning_rate": 1.9192494865055816e-05, + "loss": 7.317208314816526e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1646, + "tokens/total": 24851748, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 532140 + }, + { + "epoch": 25.734375, + "grad_norm": 3.311588807264343e-05, + "learning_rate": 1.9148541858910806e-05, + "loss": 9.501176236881292e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1647, + "tokens/total": 24866908, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 532463 + }, + { + "epoch": 25.75, + "grad_norm": 2.886946822400205e-05, + "learning_rate": 1.9104682290178246e-05, + "loss": 9.317688522969547e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1648, + "tokens/total": 24881860, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 532786 + }, + { + "epoch": 25.765625, + "grad_norm": 4.27718878199812e-05, + "learning_rate": 1.9060916273166573e-05, + "loss": 9.815552175496123e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1649, + "tokens/total": 24896916, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 533103 + }, + { + "epoch": 25.78125, + "grad_norm": 6.155242590466514e-05, + "learning_rate": 1.901724392194044e-05, + "loss": 1.283863753087644e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1650, + "tokens/total": 24912032, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 533428 + }, + { + "epoch": 25.796875, + "grad_norm": 2.0423458408913575e-05, + "learning_rate": 1.8973665350320364e-05, + "loss": 6.809571573285211e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1651, + "tokens/total": 24927176, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 533755 + }, + { + "epoch": 25.8125, + "grad_norm": 2.6709214580478147e-05, + "learning_rate": 1.8930180671882437e-05, + "loss": 9.238260076926963e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1652, + "tokens/total": 24942200, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 534078 + }, + { + "epoch": 25.828125, + "grad_norm": 2.6704970878199674e-05, + "learning_rate": 1.8886789999958054e-05, + "loss": 8.59490967286547e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1653, + "tokens/total": 24957280, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 534401 + }, + { + "epoch": 25.84375, + "grad_norm": 2.577353188826237e-05, + "learning_rate": 1.8843493447633634e-05, + "loss": 8.030106641854218e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1654, + "tokens/total": 24972348, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 534724 + }, + { + "epoch": 25.859375, + "grad_norm": 2.79900250461651e-05, + "learning_rate": 1.880029112775023e-05, + "loss": 7.53204801640095e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1655, + "tokens/total": 24987556, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 535046 + }, + { + "epoch": 25.875, + "grad_norm": 1.6696370948920958e-05, + "learning_rate": 1.8757183152903383e-05, + "loss": 5.929252324676781e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1656, + "tokens/total": 25002600, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 535369 + }, + { + "epoch": 25.890625, + "grad_norm": 3.8236645195866004e-05, + "learning_rate": 1.8714169635442687e-05, + "loss": 1.058758243743796e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1657, + "tokens/total": 25017708, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 535690 + }, + { + "epoch": 25.90625, + "grad_norm": 2.5254315914935432e-05, + "learning_rate": 1.867125068747162e-05, + "loss": 7.763784992675937e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1658, + "tokens/total": 25032856, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 536010 + }, + { + "epoch": 25.921875, + "grad_norm": 5.068371319794096e-05, + "learning_rate": 1.8628426420847094e-05, + "loss": 9.119934247792116e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1659, + "tokens/total": 25047956, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 536331 + }, + { + "epoch": 25.9375, + "grad_norm": 2.79748401226243e-05, + "learning_rate": 1.8585696947179357e-05, + "loss": 9.163549066215637e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1660, + "tokens/total": 25063100, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 536651 + }, + { + "epoch": 25.953125, + "grad_norm": 1.6841611795825884e-05, + "learning_rate": 1.854306237783154e-05, + "loss": 5.676768637385976e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1661, + "tokens/total": 25078168, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 536977 + }, + { + "epoch": 25.96875, + "grad_norm": 2.6804398657986894e-05, + "learning_rate": 1.850052282391949e-05, + "loss": 9.666181313150446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1662, + "tokens/total": 25093348, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 537296 + }, + { + "epoch": 25.984375, + "grad_norm": 2.734821282501798e-05, + "learning_rate": 1.8458078396311336e-05, + "loss": 8.552873964617902e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1663, + "tokens/total": 25108520, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 537619 + }, + { + "epoch": 26.0, + "grad_norm": 6.547004159074277e-05, + "learning_rate": 1.8415729205627368e-05, + "loss": 9.68207814366906e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1664, + "tokens/total": 25123596, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 537940 + }, + { + "epoch": 26.015625, + "grad_norm": 2.1804775315104052e-05, + "learning_rate": 1.8373475362239628e-05, + "loss": 7.388528047158616e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1665, + "tokens/total": 25138668, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 538260 + }, + { + "epoch": 26.03125, + "grad_norm": 4.50905499747023e-05, + "learning_rate": 1.8331316976271658e-05, + "loss": 1.000913016468985e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1666, + "tokens/total": 25153744, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 538585 + }, + { + "epoch": 26.046875, + "grad_norm": 3.487375215627253e-05, + "learning_rate": 1.828925415759822e-05, + "loss": 6.926044875399384e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1667, + "tokens/total": 25168668, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 538911 + }, + { + "epoch": 26.0625, + "grad_norm": 3.0830386094748974e-05, + "learning_rate": 1.8247287015845e-05, + "loss": 7.625222906426643e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1668, + "tokens/total": 25183840, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 539233 + }, + { + "epoch": 26.078125, + "grad_norm": 3.0259858249337412e-05, + "learning_rate": 1.8205415660388354e-05, + "loss": 9.191015237774991e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1669, + "tokens/total": 25199016, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 539555 + }, + { + "epoch": 26.09375, + "grad_norm": 0.00012966552458237857, + "learning_rate": 1.816364020035497e-05, + "loss": 6.5986432673526e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1670, + "tokens/total": 25214144, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 539875 + }, + { + "epoch": 26.109375, + "grad_norm": 8.401232480537146e-05, + "learning_rate": 1.812196074462161e-05, + "loss": 8.608633379481034e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1671, + "tokens/total": 25229296, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 540198 + }, + { + "epoch": 26.125, + "grad_norm": 2.6534688004176132e-05, + "learning_rate": 1.8080377401814823e-05, + "loss": 7.251278475450817e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1672, + "tokens/total": 25244320, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 540520 + }, + { + "epoch": 26.140625, + "grad_norm": 4.220080882078037e-05, + "learning_rate": 1.80388902803107e-05, + "loss": 9.777626246432192e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1673, + "tokens/total": 25259336, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 540843 + }, + { + "epoch": 26.15625, + "grad_norm": 1.678717671893537e-05, + "learning_rate": 1.7997499488234513e-05, + "loss": 5.904809086132445e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1674, + "tokens/total": 25274464, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 541167 + }, + { + "epoch": 26.171875, + "grad_norm": 5.36853913217783e-05, + "learning_rate": 1.795620513346049e-05, + "loss": 1.247278078153613e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1675, + "tokens/total": 25289548, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 541491 + }, + { + "epoch": 26.1875, + "grad_norm": 4.21668155468069e-05, + "learning_rate": 1.7915007323611526e-05, + "loss": 1.0216364216830698e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1676, + "tokens/total": 25304616, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 541809 + }, + { + "epoch": 26.203125, + "grad_norm": 2.6319510652683675e-05, + "learning_rate": 1.7873906166058917e-05, + "loss": 9.169885402116051e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1677, + "tokens/total": 25319640, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 542129 + }, + { + "epoch": 26.21875, + "grad_norm": 4.9952883273363113e-05, + "learning_rate": 1.783290176792201e-05, + "loss": 1.155791665041761e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1678, + "tokens/total": 25334800, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 542457 + }, + { + "epoch": 26.234375, + "grad_norm": 3.7695539504056796e-05, + "learning_rate": 1.779199423606802e-05, + "loss": 8.626928433841385e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1679, + "tokens/total": 25350044, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 542787 + }, + { + "epoch": 26.25, + "grad_norm": 1.952838465513196e-05, + "learning_rate": 1.775118367711167e-05, + "loss": 6.310813773779955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1680, + "tokens/total": 25365064, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 543108 + }, + { + "epoch": 26.265625, + "grad_norm": 2.27326963795349e-05, + "learning_rate": 1.7710470197414998e-05, + "loss": 7.053425292724569e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1681, + "tokens/total": 25380168, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 543434 + }, + { + "epoch": 26.28125, + "grad_norm": 2.4406215743510984e-05, + "learning_rate": 1.766985390308697e-05, + "loss": 7.919386462162947e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1682, + "tokens/total": 25395220, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 543753 + }, + { + "epoch": 26.296875, + "grad_norm": 6.535748980240896e-05, + "learning_rate": 1.7629334899983308e-05, + "loss": 1.4268932773120468e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1683, + "tokens/total": 25410384, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 544076 + }, + { + "epoch": 26.3125, + "grad_norm": 6.015961480443366e-05, + "learning_rate": 1.7588913293706143e-05, + "loss": 1.219592718371132e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1684, + "tokens/total": 25425476, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 544405 + }, + { + "epoch": 26.328125, + "grad_norm": 1.8175684090238065e-05, + "learning_rate": 1.754858918960378e-05, + "loss": 6.112337587182992e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1685, + "tokens/total": 25440588, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 544730 + }, + { + "epoch": 26.34375, + "grad_norm": 1.8276328773936257e-05, + "learning_rate": 1.7508362692770442e-05, + "loss": 6.11620578183647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1686, + "tokens/total": 25455748, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 545054 + }, + { + "epoch": 26.359375, + "grad_norm": 2.2596565031562932e-05, + "learning_rate": 1.7468233908045872e-05, + "loss": 7.535849135820172e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1687, + "tokens/total": 25470876, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 545378 + }, + { + "epoch": 26.375, + "grad_norm": 1.860904558270704e-05, + "learning_rate": 1.742820294001526e-05, + "loss": 6.162229624351312e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1688, + "tokens/total": 25485920, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 545701 + }, + { + "epoch": 26.390625, + "grad_norm": 2.5106633984250948e-05, + "learning_rate": 1.738826989300879e-05, + "loss": 7.894785198914178e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1689, + "tokens/total": 25501072, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 546027 + }, + { + "epoch": 26.40625, + "grad_norm": 3.202988955308683e-05, + "learning_rate": 1.7348434871101478e-05, + "loss": 6.271506549637706e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1690, + "tokens/total": 25516172, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 546349 + }, + { + "epoch": 26.421875, + "grad_norm": 2.8955957532161847e-05, + "learning_rate": 1.7308697978112828e-05, + "loss": 8.948398999564233e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1691, + "tokens/total": 25531392, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 546670 + }, + { + "epoch": 26.4375, + "grad_norm": 1.8187483874498866e-05, + "learning_rate": 1.7269059317606643e-05, + "loss": 6.350086891870887e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1692, + "tokens/total": 25546420, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 546990 + }, + { + "epoch": 26.453125, + "grad_norm": 1.9787221390288323e-05, + "learning_rate": 1.7229518992890666e-05, + "loss": 6.573193331860239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1693, + "tokens/total": 25561496, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 547314 + }, + { + "epoch": 26.46875, + "grad_norm": 1.773746589606162e-05, + "learning_rate": 1.719007710701639e-05, + "loss": 5.96974814470741e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1694, + "tokens/total": 25576600, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 547638 + }, + { + "epoch": 26.484375, + "grad_norm": 1.9584416804718785e-05, + "learning_rate": 1.715073376277871e-05, + "loss": 6.578463285222824e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1695, + "tokens/total": 25591720, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 547965 + }, + { + "epoch": 26.5, + "grad_norm": 2.8467964511946775e-05, + "learning_rate": 1.711148906271574e-05, + "loss": 9.086448926609592e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1696, + "tokens/total": 25606872, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 548293 + }, + { + "epoch": 26.515625, + "grad_norm": 3.497996658552438e-05, + "learning_rate": 1.707234310910849e-05, + "loss": 7.713273930676223e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1697, + "tokens/total": 25622016, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 548615 + }, + { + "epoch": 26.53125, + "grad_norm": 1.987997347896453e-05, + "learning_rate": 1.7033296003980595e-05, + "loss": 6.667286811534723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1698, + "tokens/total": 25637196, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 548939 + }, + { + "epoch": 26.546875, + "grad_norm": 2.2826592612545937e-05, + "learning_rate": 1.6994347849098084e-05, + "loss": 7.605228802276542e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1699, + "tokens/total": 25652400, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 549265 + }, + { + "epoch": 26.5625, + "grad_norm": 2.337392834306229e-05, + "learning_rate": 1.695549874596911e-05, + "loss": 7.210069270513486e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1700, + "tokens/total": 25667476, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 549589 + }, + { + "epoch": 26.578125, + "grad_norm": 7.547111454186961e-05, + "learning_rate": 1.6916748795843658e-05, + "loss": 1.398616063852387e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1701, + "tokens/total": 25682560, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 549908 + }, + { + "epoch": 26.59375, + "grad_norm": 3.018978713953402e-05, + "learning_rate": 1.6878098099713293e-05, + "loss": 9.701298040454276e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1702, + "tokens/total": 25697584, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 550231 + }, + { + "epoch": 26.609375, + "grad_norm": 2.8210704840603285e-05, + "learning_rate": 1.6839546758310904e-05, + "loss": 9.198062684845354e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1703, + "tokens/total": 25712736, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 550559 + }, + { + "epoch": 26.625, + "grad_norm": 3.303947232780047e-05, + "learning_rate": 1.680109487211043e-05, + "loss": 6.347120233840542e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1704, + "tokens/total": 25727804, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 550884 + }, + { + "epoch": 26.640625, + "grad_norm": 4.978870492777787e-05, + "learning_rate": 1.6762742541326647e-05, + "loss": 7.038178750917723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1705, + "tokens/total": 25742828, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 551205 + }, + { + "epoch": 26.65625, + "grad_norm": 2.543500522733666e-05, + "learning_rate": 1.6724489865914805e-05, + "loss": 6.919168527019792e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1706, + "tokens/total": 25757964, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 551527 + }, + { + "epoch": 26.671875, + "grad_norm": 4.568801887216978e-05, + "learning_rate": 1.668633694557048e-05, + "loss": 8.809393534647825e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1707, + "tokens/total": 25772992, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 551852 + }, + { + "epoch": 26.6875, + "grad_norm": 0.00014668403309769928, + "learning_rate": 1.6648283879729217e-05, + "loss": 2.3837603748688707e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1708, + "tokens/total": 25787984, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 552179 + }, + { + "epoch": 26.703125, + "grad_norm": 1.9564558897400275e-05, + "learning_rate": 1.661033076756638e-05, + "loss": 6.813008894823724e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1709, + "tokens/total": 25803048, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 552502 + }, + { + "epoch": 26.71875, + "grad_norm": 2.45740229729563e-05, + "learning_rate": 1.6572477707996746e-05, + "loss": 8.650473546367721e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1710, + "tokens/total": 25818120, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 552826 + }, + { + "epoch": 26.734375, + "grad_norm": 2.84390443994198e-05, + "learning_rate": 1.6534724799674412e-05, + "loss": 1.0375712236054824e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1711, + "tokens/total": 25833148, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 553149 + }, + { + "epoch": 26.75, + "grad_norm": 2.72775323537644e-05, + "learning_rate": 1.6497072140992386e-05, + "loss": 9.775357057151268e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1712, + "tokens/total": 25848284, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 553472 + }, + { + "epoch": 26.765625, + "grad_norm": 2.2793297830503434e-05, + "learning_rate": 1.6459519830082486e-05, + "loss": 7.153516889957245e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.01, + "memory/max_allocated (GiB)": 33.01, + "ppl": 1.0, + "step": 1713, + "tokens/total": 25863464, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 553796 + }, + { + "epoch": 26.78125, + "grad_norm": 3.541313708410598e-05, + "learning_rate": 1.64220679648149e-05, + "loss": 9.147520358965266e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1714, + "tokens/total": 25878620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 554122 + }, + { + "epoch": 26.796875, + "grad_norm": 2.4762030079727992e-05, + "learning_rate": 1.6384716642798118e-05, + "loss": 8.071201591519639e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1715, + "tokens/total": 25893648, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 554438 + }, + { + "epoch": 26.8125, + "grad_norm": 3.765181827475317e-05, + "learning_rate": 1.6347465961378533e-05, + "loss": 9.889246257444029e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1716, + "tokens/total": 25908836, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 554761 + }, + { + "epoch": 26.828125, + "grad_norm": 3.4304011933272704e-05, + "learning_rate": 1.6310316017640277e-05, + "loss": 9.404269007973198e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1717, + "tokens/total": 25923892, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 555084 + }, + { + "epoch": 26.84375, + "grad_norm": 1.6279673218377866e-05, + "learning_rate": 1.6273266908404898e-05, + "loss": 5.384383712225826e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1718, + "tokens/total": 25939004, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 555408 + }, + { + "epoch": 26.859375, + "grad_norm": 2.009047966566868e-05, + "learning_rate": 1.623631873023121e-05, + "loss": 5.994839398226759e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1719, + "tokens/total": 25954084, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 555727 + }, + { + "epoch": 26.875, + "grad_norm": 1.92499028344173e-05, + "learning_rate": 1.6199471579414924e-05, + "loss": 5.929538247073651e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1720, + "tokens/total": 25969136, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 556050 + }, + { + "epoch": 26.890625, + "grad_norm": 3.588489562389441e-05, + "learning_rate": 1.6162725551988453e-05, + "loss": 9.247119692190608e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1721, + "tokens/total": 25984236, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 556372 + }, + { + "epoch": 26.90625, + "grad_norm": 1.9477341993479058e-05, + "learning_rate": 1.612608074372068e-05, + "loss": 6.409346724467468e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1722, + "tokens/total": 25999280, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 556695 + }, + { + "epoch": 26.921875, + "grad_norm": 5.339880590327084e-05, + "learning_rate": 1.608953725011667e-05, + "loss": 1.0675923931557918e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1723, + "tokens/total": 26014452, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 557016 + }, + { + "epoch": 26.9375, + "grad_norm": 2.8406186174834147e-05, + "learning_rate": 1.6053095166417448e-05, + "loss": 7.62034403578582e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1724, + "tokens/total": 26029416, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 557338 + }, + { + "epoch": 26.953125, + "grad_norm": 1.9125778635498136e-05, + "learning_rate": 1.6016754587599742e-05, + "loss": 6.869717026347644e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1725, + "tokens/total": 26044444, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 557661 + }, + { + "epoch": 26.96875, + "grad_norm": 3.1367297196993604e-05, + "learning_rate": 1.5980515608375717e-05, + "loss": 9.762167110238806e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1726, + "tokens/total": 26059536, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 557987 + }, + { + "epoch": 26.984375, + "grad_norm": 5.655536369886249e-05, + "learning_rate": 1.594437832319276e-05, + "loss": 1.3101063132126e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1727, + "tokens/total": 26074684, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 558309 + }, + { + "epoch": 27.0, + "grad_norm": 1.9413802874623798e-05, + "learning_rate": 1.590834282623324e-05, + "loss": 6.8056351665291e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1728, + "tokens/total": 26089776, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 558630 + }, + { + "epoch": 27.015625, + "grad_norm": 2.938322904810775e-05, + "learning_rate": 1.587240921141418e-05, + "loss": 9.43116845064651e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1729, + "tokens/total": 26104836, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 558952 + }, + { + "epoch": 27.03125, + "grad_norm": 2.5613642719690688e-05, + "learning_rate": 1.5836577572387136e-05, + "loss": 8.121973564811924e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1730, + "tokens/total": 26119808, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 559276 + }, + { + "epoch": 27.046875, + "grad_norm": 2.3899920051917434e-05, + "learning_rate": 1.5800848002537854e-05, + "loss": 7.85814847859001e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1731, + "tokens/total": 26135008, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 559602 + }, + { + "epoch": 27.0625, + "grad_norm": 1.6668673197273165e-05, + "learning_rate": 1.5765220594986108e-05, + "loss": 4.77505693652347e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1732, + "tokens/total": 26150168, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 559928 + }, + { + "epoch": 27.078125, + "grad_norm": 1.8357168301008642e-05, + "learning_rate": 1.572969544258534e-05, + "loss": 6.44639840174932e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1733, + "tokens/total": 26165200, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 560253 + }, + { + "epoch": 27.09375, + "grad_norm": 2.1770381863461807e-05, + "learning_rate": 1.5694272637922552e-05, + "loss": 7.405135988847178e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1734, + "tokens/total": 26180276, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 560573 + }, + { + "epoch": 27.109375, + "grad_norm": 4.56554043921642e-05, + "learning_rate": 1.565895227331799e-05, + "loss": 1.0226717677142005e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1735, + "tokens/total": 26195316, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 560901 + }, + { + "epoch": 27.125, + "grad_norm": 2.9731074391747825e-05, + "learning_rate": 1.5623734440824894e-05, + "loss": 9.048381457432697e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1736, + "tokens/total": 26210372, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 561229 + }, + { + "epoch": 27.140625, + "grad_norm": 2.5160285076708533e-05, + "learning_rate": 1.5588619232229302e-05, + "loss": 7.900587206677301e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1737, + "tokens/total": 26225508, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 561553 + }, + { + "epoch": 27.15625, + "grad_norm": 4.873782381764613e-05, + "learning_rate": 1.5553606739049802e-05, + "loss": 1.1325442983434186e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1738, + "tokens/total": 26240604, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 561875 + }, + { + "epoch": 27.171875, + "grad_norm": 1.83929023478413e-05, + "learning_rate": 1.5518697052537252e-05, + "loss": 5.327599410520634e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1739, + "tokens/total": 26255788, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 562200 + }, + { + "epoch": 27.1875, + "grad_norm": 2.970745299535338e-05, + "learning_rate": 1.548389026367458e-05, + "loss": 6.772106644348241e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1740, + "tokens/total": 26270832, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 562521 + }, + { + "epoch": 27.203125, + "grad_norm": 5.0773236580425873e-05, + "learning_rate": 1.5449186463176544e-05, + "loss": 9.112213206208253e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1741, + "tokens/total": 26285940, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 562839 + }, + { + "epoch": 27.21875, + "grad_norm": 2.385601692367345e-05, + "learning_rate": 1.5414585741489483e-05, + "loss": 6.853856007182912e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1742, + "tokens/total": 26301072, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 563163 + }, + { + "epoch": 27.234375, + "grad_norm": 2.8454616767703556e-05, + "learning_rate": 1.5380088188791114e-05, + "loss": 8.334857284353347e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1743, + "tokens/total": 26316136, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 563484 + }, + { + "epoch": 27.25, + "grad_norm": 2.532713915570639e-05, + "learning_rate": 1.5345693894990237e-05, + "loss": 7.942676347738598e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1744, + "tokens/total": 26331332, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 563808 + }, + { + "epoch": 27.265625, + "grad_norm": 1.883327786345035e-05, + "learning_rate": 1.531140294972655e-05, + "loss": 6.613986442971509e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1745, + "tokens/total": 26346464, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 564129 + }, + { + "epoch": 27.28125, + "grad_norm": 1.993072874029167e-05, + "learning_rate": 1.5277215442370385e-05, + "loss": 6.575228326255456e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1746, + "tokens/total": 26361668, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 564452 + }, + { + "epoch": 27.296875, + "grad_norm": 2.1668802219210193e-05, + "learning_rate": 1.524313146202252e-05, + "loss": 7.00367991157691e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1747, + "tokens/total": 26376888, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 564775 + }, + { + "epoch": 27.3125, + "grad_norm": 4.106778942514211e-05, + "learning_rate": 1.5209151097513898e-05, + "loss": 9.947225407813676e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1748, + "tokens/total": 26391924, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 565093 + }, + { + "epoch": 27.328125, + "grad_norm": 2.6749810785986483e-05, + "learning_rate": 1.5175274437405423e-05, + "loss": 7.956282956911309e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1749, + "tokens/total": 26407012, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 565415 + }, + { + "epoch": 27.34375, + "grad_norm": 7.091330917319283e-05, + "learning_rate": 1.5141501569987693e-05, + "loss": 1.5505597730225418e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1750, + "tokens/total": 26422096, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 565734 + }, + { + "epoch": 27.359375, + "grad_norm": 6.254486652323976e-05, + "learning_rate": 1.510783258328084e-05, + "loss": 8.418347761107725e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1751, + "tokens/total": 26437164, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 566061 + }, + { + "epoch": 27.375, + "grad_norm": 7.242299761855975e-05, + "learning_rate": 1.5074267565034234e-05, + "loss": 1.0018077318818541e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1752, + "tokens/total": 26452252, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 566386 + }, + { + "epoch": 27.390625, + "grad_norm": 1.6571015294175595e-05, + "learning_rate": 1.5040806602726277e-05, + "loss": 5.233288788986101e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1753, + "tokens/total": 26467400, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 566710 + }, + { + "epoch": 27.40625, + "grad_norm": 1.479751335864421e-05, + "learning_rate": 1.5007449783564184e-05, + "loss": 5.166459686734015e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1754, + "tokens/total": 26482648, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 567034 + }, + { + "epoch": 27.421875, + "grad_norm": 2.060231054201722e-05, + "learning_rate": 1.4974197194483742e-05, + "loss": 6.929800520083518e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1755, + "tokens/total": 26497772, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 567362 + }, + { + "epoch": 27.4375, + "grad_norm": 2.6620306016411632e-05, + "learning_rate": 1.4941048922149097e-05, + "loss": 8.837892551127879e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1756, + "tokens/total": 26512856, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 567682 + }, + { + "epoch": 27.453125, + "grad_norm": 2.9592554710689e-05, + "learning_rate": 1.4908005052952528e-05, + "loss": 8.854682960190985e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1757, + "tokens/total": 26527988, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 568004 + }, + { + "epoch": 27.46875, + "grad_norm": 6.27923509455286e-05, + "learning_rate": 1.487506567301419e-05, + "loss": 1.4328117003969965e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1758, + "tokens/total": 26543248, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 568330 + }, + { + "epoch": 27.484375, + "grad_norm": 1.7339458281639963e-05, + "learning_rate": 1.4842230868181925e-05, + "loss": 6.088740747145494e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1759, + "tokens/total": 26558268, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 568653 + }, + { + "epoch": 27.5, + "grad_norm": 2.135582508344669e-05, + "learning_rate": 1.4809500724031056e-05, + "loss": 7.239761998789618e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1760, + "tokens/total": 26573296, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 568976 + }, + { + "epoch": 27.515625, + "grad_norm": 2.5403491235920228e-05, + "learning_rate": 1.477687532586407e-05, + "loss": 8.747913398110541e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1761, + "tokens/total": 26588388, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 569300 + }, + { + "epoch": 27.53125, + "grad_norm": 2.328540722373873e-05, + "learning_rate": 1.4744354758710538e-05, + "loss": 6.876676934552961e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1762, + "tokens/total": 26603384, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 569626 + }, + { + "epoch": 27.546875, + "grad_norm": 3.077424116781913e-05, + "learning_rate": 1.4711939107326761e-05, + "loss": 6.319863814496784e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1763, + "tokens/total": 26618504, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 569953 + }, + { + "epoch": 27.5625, + "grad_norm": 2.1148651285329834e-05, + "learning_rate": 1.4679628456195632e-05, + "loss": 6.30844340321346e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1764, + "tokens/total": 26633596, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 570275 + }, + { + "epoch": 27.578125, + "grad_norm": 3.40379228873644e-05, + "learning_rate": 1.464742288952636e-05, + "loss": 1.0213068435405148e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1765, + "tokens/total": 26648724, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 570599 + }, + { + "epoch": 27.59375, + "grad_norm": 4.755011832457967e-05, + "learning_rate": 1.4615322491254324e-05, + "loss": 8.87851797415351e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1766, + "tokens/total": 26663768, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 570922 + }, + { + "epoch": 27.609375, + "grad_norm": 2.036234764091205e-05, + "learning_rate": 1.4583327345040779e-05, + "loss": 6.524906552840548e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1767, + "tokens/total": 26678732, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 571243 + }, + { + "epoch": 27.625, + "grad_norm": 2.8430495149223134e-05, + "learning_rate": 1.4551437534272663e-05, + "loss": 9.370983207190875e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1768, + "tokens/total": 26693984, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 571565 + }, + { + "epoch": 27.640625, + "grad_norm": 4.365336280898191e-05, + "learning_rate": 1.4519653142062385e-05, + "loss": 7.276520364030148e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1769, + "tokens/total": 26709048, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 571888 + }, + { + "epoch": 27.65625, + "grad_norm": 2.7982341634924524e-05, + "learning_rate": 1.4487974251247646e-05, + "loss": 7.913689046290528e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1770, + "tokens/total": 26724236, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 572212 + }, + { + "epoch": 27.671875, + "grad_norm": 2.0264667909941636e-05, + "learning_rate": 1.4456400944391146e-05, + "loss": 6.552739932885743e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1771, + "tokens/total": 26739400, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 572535 + }, + { + "epoch": 27.6875, + "grad_norm": 1.957043969014194e-05, + "learning_rate": 1.4424933303780411e-05, + "loss": 6.316288931884628e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1772, + "tokens/total": 26754612, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 572852 + }, + { + "epoch": 27.703125, + "grad_norm": 3.130604818579741e-05, + "learning_rate": 1.439357141142759e-05, + "loss": 7.865908742132888e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1773, + "tokens/total": 26769748, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 573179 + }, + { + "epoch": 27.71875, + "grad_norm": 2.5651248506619595e-05, + "learning_rate": 1.4362315349069214e-05, + "loss": 7.447280836458958e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1774, + "tokens/total": 26784856, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 573501 + }, + { + "epoch": 27.734375, + "grad_norm": 6.596439197892323e-05, + "learning_rate": 1.4331165198166013e-05, + "loss": 1.1402089512557723e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1775, + "tokens/total": 26799916, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 573825 + }, + { + "epoch": 27.75, + "grad_norm": 3.0464654628303833e-05, + "learning_rate": 1.4300121039902682e-05, + "loss": 9.442613873034134e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1776, + "tokens/total": 26815020, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 574145 + }, + { + "epoch": 27.765625, + "grad_norm": 4.0536036976845935e-05, + "learning_rate": 1.4269182955187651e-05, + "loss": 9.041528983289027e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1777, + "tokens/total": 26830080, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 574467 + }, + { + "epoch": 27.78125, + "grad_norm": 2.013207813433837e-05, + "learning_rate": 1.4238351024652918e-05, + "loss": 6.537012495755334e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1778, + "tokens/total": 26845148, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 574791 + }, + { + "epoch": 27.796875, + "grad_norm": 2.2423595510190353e-05, + "learning_rate": 1.4207625328653836e-05, + "loss": 5.749410547650768e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1779, + "tokens/total": 26860380, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 575113 + }, + { + "epoch": 27.8125, + "grad_norm": 5.4138821724336594e-05, + "learning_rate": 1.4177005947268834e-05, + "loss": 9.616329634809517e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1780, + "tokens/total": 26875444, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 575433 + }, + { + "epoch": 27.828125, + "grad_norm": 2.6146226446144283e-05, + "learning_rate": 1.414649296029931e-05, + "loss": 7.86516466178e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1781, + "tokens/total": 26890548, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 575758 + }, + { + "epoch": 27.84375, + "grad_norm": 2.6425903342897072e-05, + "learning_rate": 1.4116086447269341e-05, + "loss": 6.951200930416235e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1782, + "tokens/total": 26905592, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 576081 + }, + { + "epoch": 27.859375, + "grad_norm": 2.5505822122795507e-05, + "learning_rate": 1.4085786487425545e-05, + "loss": 8.269696536444826e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1783, + "tokens/total": 26920684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 576408 + }, + { + "epoch": 27.875, + "grad_norm": 1.9805660485872068e-05, + "learning_rate": 1.405559315973677e-05, + "loss": 6.406462489394471e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1784, + "tokens/total": 26935808, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 576732 + }, + { + "epoch": 27.890625, + "grad_norm": 1.7157317415694706e-05, + "learning_rate": 1.4025506542894032e-05, + "loss": 5.960260978099541e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1785, + "tokens/total": 26950860, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 577056 + }, + { + "epoch": 27.90625, + "grad_norm": 1.6730873539927416e-05, + "learning_rate": 1.3995526715310176e-05, + "loss": 5.397500331127958e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1786, + "tokens/total": 26965972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 577383 + }, + { + "epoch": 27.921875, + "grad_norm": 3.233224924770184e-05, + "learning_rate": 1.3965653755119778e-05, + "loss": 8.743973012315109e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1787, + "tokens/total": 26981100, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 577708 + }, + { + "epoch": 27.9375, + "grad_norm": 5.368107667891309e-05, + "learning_rate": 1.3935887740178832e-05, + "loss": 1.2256308536962024e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1788, + "tokens/total": 26996160, + "tokens/train_per_sec_per_gpu": 8.94, + "tokens/trainable": 578030 + }, + { + "epoch": 27.953125, + "grad_norm": 1.7229285731445998e-05, + "learning_rate": 1.3906228748064665e-05, + "loss": 6.102958991505147e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1789, + "tokens/total": 27011240, + "tokens/train_per_sec_per_gpu": 8.92, + "tokens/trainable": 578350 + }, + { + "epoch": 27.96875, + "grad_norm": 1.4387224837264512e-05, + "learning_rate": 1.3876676856075644e-05, + "loss": 4.741381189887761e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1790, + "tokens/total": 27026260, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 578673 + }, + { + "epoch": 27.984375, + "grad_norm": 5.2459097787505016e-05, + "learning_rate": 1.3847232141231017e-05, + "loss": 5.937375249231991e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1791, + "tokens/total": 27041364, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 578995 + }, + { + "epoch": 28.0, + "grad_norm": 0.00013888241664972156, + "learning_rate": 1.381789468027069e-05, + "loss": 2.0574227619363228e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1792, + "tokens/total": 27056476, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 579320 + }, + { + "epoch": 28.015625, + "grad_norm": 2.44708098762203e-05, + "learning_rate": 1.3788664549655077e-05, + "loss": 8.169480452124844e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1793, + "tokens/total": 27071592, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 579645 + }, + { + "epoch": 28.03125, + "grad_norm": 3.30656512232963e-05, + "learning_rate": 1.3759541825564826e-05, + "loss": 9.484371048529283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1794, + "tokens/total": 27086620, + "tokens/train_per_sec_per_gpu": 10.11, + "tokens/trainable": 579970 + }, + { + "epoch": 28.046875, + "grad_norm": 1.6414061974501237e-05, + "learning_rate": 1.3730526583900671e-05, + "loss": 5.516439500752313e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1795, + "tokens/total": 27101828, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 580294 + }, + { + "epoch": 28.0625, + "grad_norm": 1.9054350559599698e-05, + "learning_rate": 1.3701618900283219e-05, + "loss": 6.071609846003412e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1796, + "tokens/total": 27116992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 580618 + }, + { + "epoch": 28.078125, + "grad_norm": 2.8527672839118168e-05, + "learning_rate": 1.3672818850052752e-05, + "loss": 7.596701721013233e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1797, + "tokens/total": 27132184, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 580946 + }, + { + "epoch": 28.09375, + "grad_norm": 1.8931648810394108e-05, + "learning_rate": 1.3644126508269059e-05, + "loss": 6.388930842149421e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1798, + "tokens/total": 27147260, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 581273 + }, + { + "epoch": 28.109375, + "grad_norm": 0.00014350400306284428, + "learning_rate": 1.3615541949711171e-05, + "loss": 2.175885128963273e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1799, + "tokens/total": 27162400, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 581595 + }, + { + "epoch": 28.125, + "grad_norm": 3.45108455803711e-05, + "learning_rate": 1.3587065248877252e-05, + "loss": 9.200271620102285e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1800, + "tokens/total": 27177428, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 581914 + }, + { + "epoch": 28.140625, + "grad_norm": 7.43305281503126e-05, + "learning_rate": 1.3558696479984334e-05, + "loss": 1.4313710607893881e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1801, + "tokens/total": 27192596, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 582236 + }, + { + "epoch": 28.15625, + "grad_norm": 2.695326656976249e-05, + "learning_rate": 1.3530435716968188e-05, + "loss": 8.167522196345089e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1802, + "tokens/total": 27207772, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 582556 + }, + { + "epoch": 28.171875, + "grad_norm": 2.958078039227985e-05, + "learning_rate": 1.3502283033483032e-05, + "loss": 8.974394063443469e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1803, + "tokens/total": 27222944, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 582880 + }, + { + "epoch": 28.1875, + "grad_norm": 2.210792990808841e-05, + "learning_rate": 1.3474238502901482e-05, + "loss": 6.395144396265096e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1804, + "tokens/total": 27238096, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 583202 + }, + { + "epoch": 28.203125, + "grad_norm": 3.3400221582269296e-05, + "learning_rate": 1.3446302198314217e-05, + "loss": 6.919772772562283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1805, + "tokens/total": 27253088, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 583518 + }, + { + "epoch": 28.21875, + "grad_norm": 4.2061234125867486e-05, + "learning_rate": 1.3418474192529914e-05, + "loss": 9.801267424336402e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1806, + "tokens/total": 27268100, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 583846 + }, + { + "epoch": 28.234375, + "grad_norm": 4.1669027268653736e-05, + "learning_rate": 1.339075455807492e-05, + "loss": 1.131160956902022e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1807, + "tokens/total": 27283176, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 584165 + }, + { + "epoch": 28.25, + "grad_norm": 4.1913855966413394e-05, + "learning_rate": 1.3363143367193213e-05, + "loss": 1.1952853355978732e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1808, + "tokens/total": 27298308, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 584484 + }, + { + "epoch": 28.265625, + "grad_norm": 2.317908001714386e-05, + "learning_rate": 1.3335640691846104e-05, + "loss": 7.839843192414264e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1809, + "tokens/total": 27313564, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 584808 + }, + { + "epoch": 28.28125, + "grad_norm": 3.1093062716536224e-05, + "learning_rate": 1.3308246603712101e-05, + "loss": 7.18230978691281e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1810, + "tokens/total": 27328724, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 585131 + }, + { + "epoch": 28.296875, + "grad_norm": 0.0003394655359443277, + "learning_rate": 1.3280961174186685e-05, + "loss": 7.776154120620049e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1811, + "tokens/total": 27343736, + "tokens/train_per_sec_per_gpu": 10.16, + "tokens/trainable": 585456 + }, + { + "epoch": 28.3125, + "grad_norm": 1.848299871198833e-05, + "learning_rate": 1.3253784474382182e-05, + "loss": 6.254117010939808e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1812, + "tokens/total": 27358912, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 585785 + }, + { + "epoch": 28.328125, + "grad_norm": 2.4372886400669813e-05, + "learning_rate": 1.3226716575127523e-05, + "loss": 7.557384265055589e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1813, + "tokens/total": 27373972, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 586112 + }, + { + "epoch": 28.34375, + "grad_norm": 1.868056097009685e-05, + "learning_rate": 1.3199757546968072e-05, + "loss": 6.250833166632219e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1814, + "tokens/total": 27389104, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 586436 + }, + { + "epoch": 28.359375, + "grad_norm": 1.9191456885891967e-05, + "learning_rate": 1.317290746016546e-05, + "loss": 6.077988246033783e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1815, + "tokens/total": 27404256, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 586760 + }, + { + "epoch": 28.375, + "grad_norm": 1.9965751562267542e-05, + "learning_rate": 1.3146166384697389e-05, + "loss": 7.22984168533003e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1816, + "tokens/total": 27419320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 587081 + }, + { + "epoch": 28.390625, + "grad_norm": 2.7831747502204962e-05, + "learning_rate": 1.311953439025745e-05, + "loss": 7.469120646419469e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1817, + "tokens/total": 27434472, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 587403 + }, + { + "epoch": 28.40625, + "grad_norm": 2.5066614398383535e-05, + "learning_rate": 1.3093011546254953e-05, + "loss": 8.392738664042554e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1818, + "tokens/total": 27449492, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 587723 + }, + { + "epoch": 28.421875, + "grad_norm": 2.3608259652974084e-05, + "learning_rate": 1.306659792181472e-05, + "loss": 6.627183211094234e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1819, + "tokens/total": 27464616, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 588045 + }, + { + "epoch": 28.4375, + "grad_norm": 1.733022327243816e-05, + "learning_rate": 1.3040293585776916e-05, + "loss": 5.581929372056038e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1820, + "tokens/total": 27479816, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 588371 + }, + { + "epoch": 28.453125, + "grad_norm": 1.7555565136717632e-05, + "learning_rate": 1.3014098606696909e-05, + "loss": 6.149463729343552e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1821, + "tokens/total": 27495072, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 588694 + }, + { + "epoch": 28.46875, + "grad_norm": 1.9636601791717112e-05, + "learning_rate": 1.2988013052845022e-05, + "loss": 5.928602035965014e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1822, + "tokens/total": 27510196, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 589016 + }, + { + "epoch": 28.484375, + "grad_norm": 2.9902254027547315e-05, + "learning_rate": 1.2962036992206406e-05, + "loss": 9.215835348186374e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1823, + "tokens/total": 27525296, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 589335 + }, + { + "epoch": 28.5, + "grad_norm": 1.8362854461884126e-05, + "learning_rate": 1.2936170492480828e-05, + "loss": 6.231237534848333e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1824, + "tokens/total": 27540288, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 589661 + }, + { + "epoch": 28.515625, + "grad_norm": 4.684385930886492e-05, + "learning_rate": 1.2910413621082551e-05, + "loss": 8.322127769133658e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1825, + "tokens/total": 27555456, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 589986 + }, + { + "epoch": 28.53125, + "grad_norm": 2.6542498744674958e-05, + "learning_rate": 1.2884766445140084e-05, + "loss": 7.261065775310271e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1826, + "tokens/total": 27570532, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 590311 + }, + { + "epoch": 28.546875, + "grad_norm": 1.8901928342529573e-05, + "learning_rate": 1.2859229031496064e-05, + "loss": 6.36762194972107e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1827, + "tokens/total": 27585548, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 590634 + }, + { + "epoch": 28.5625, + "grad_norm": 6.673804455203936e-05, + "learning_rate": 1.2833801446707045e-05, + "loss": 9.656648671807488e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1828, + "tokens/total": 27600660, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 590960 + }, + { + "epoch": 28.578125, + "grad_norm": 5.063611388322897e-05, + "learning_rate": 1.2808483757043355e-05, + "loss": 9.7301278856321e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1829, + "tokens/total": 27615664, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 591285 + }, + { + "epoch": 28.59375, + "grad_norm": 1.8552438632468693e-05, + "learning_rate": 1.2783276028488907e-05, + "loss": 5.71606562971283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1830, + "tokens/total": 27630736, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 591606 + }, + { + "epoch": 28.609375, + "grad_norm": 1.9527142285369337e-05, + "learning_rate": 1.2758178326741027e-05, + "loss": 6.057848622731399e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1831, + "tokens/total": 27645796, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 591931 + }, + { + "epoch": 28.625, + "grad_norm": 1.6779551515355706e-05, + "learning_rate": 1.2733190717210284e-05, + "loss": 5.508014737642952e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1832, + "tokens/total": 27660764, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 592251 + }, + { + "epoch": 28.640625, + "grad_norm": 2.4373117412324063e-05, + "learning_rate": 1.270831326502032e-05, + "loss": 7.792164637976384e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1833, + "tokens/total": 27675848, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 592571 + }, + { + "epoch": 28.65625, + "grad_norm": 4.453696237760596e-05, + "learning_rate": 1.2683546035007688e-05, + "loss": 4.933323793920863e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1834, + "tokens/total": 27690904, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 592896 + }, + { + "epoch": 28.671875, + "grad_norm": 1.8245285900775343e-05, + "learning_rate": 1.2658889091721654e-05, + "loss": 5.746936722061946e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1835, + "tokens/total": 27705948, + "tokens/train_per_sec_per_gpu": 8.75, + "tokens/trainable": 593221 + }, + { + "epoch": 28.6875, + "grad_norm": 4.028835246572271e-05, + "learning_rate": 1.2634342499424082e-05, + "loss": 1.0438295703352196e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1836, + "tokens/total": 27721120, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 593548 + }, + { + "epoch": 28.703125, + "grad_norm": 5.2887357014697045e-05, + "learning_rate": 1.260990632208922e-05, + "loss": 6.895651836202887e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1837, + "tokens/total": 27736312, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 593872 + }, + { + "epoch": 28.71875, + "grad_norm": 2.9709606678807177e-05, + "learning_rate": 1.2585580623403544e-05, + "loss": 6.881452918605646e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1838, + "tokens/total": 27751464, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 594196 + }, + { + "epoch": 28.734375, + "grad_norm": 1.9993198293377645e-05, + "learning_rate": 1.2561365466765592e-05, + "loss": 6.773385052838421e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1839, + "tokens/total": 27766576, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 594516 + }, + { + "epoch": 28.75, + "grad_norm": 2.1794732674607076e-05, + "learning_rate": 1.253726091528583e-05, + "loss": 5.814336532239395e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1840, + "tokens/total": 27781724, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 594840 + }, + { + "epoch": 28.765625, + "grad_norm": 2.2327696569846012e-05, + "learning_rate": 1.251326703178643e-05, + "loss": 6.558427685376955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1841, + "tokens/total": 27796844, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 595166 + }, + { + "epoch": 28.78125, + "grad_norm": 2.933776704594493e-05, + "learning_rate": 1.2489383878801151e-05, + "loss": 7.807942665749579e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1842, + "tokens/total": 27811944, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 595490 + }, + { + "epoch": 28.796875, + "grad_norm": 1.8162792912335135e-05, + "learning_rate": 1.246561151857516e-05, + "loss": 6.303477562141779e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1843, + "tokens/total": 27826968, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 595812 + }, + { + "epoch": 28.8125, + "grad_norm": 2.2704663933836855e-05, + "learning_rate": 1.244195001306488e-05, + "loss": 6.590962584596127e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1844, + "tokens/total": 27842012, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 596134 + }, + { + "epoch": 28.828125, + "grad_norm": 1.8370257748756558e-05, + "learning_rate": 1.2418399423937805e-05, + "loss": 5.426193183666328e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1845, + "tokens/total": 27857132, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 596456 + }, + { + "epoch": 28.84375, + "grad_norm": 5.850333400303498e-05, + "learning_rate": 1.2394959812572368e-05, + "loss": 7.616698667334276e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1846, + "tokens/total": 27872212, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 596783 + }, + { + "epoch": 28.859375, + "grad_norm": 2.45760238613002e-05, + "learning_rate": 1.2371631240057752e-05, + "loss": 7.556487844340154e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1847, + "tokens/total": 27887348, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 597107 + }, + { + "epoch": 28.875, + "grad_norm": 2.045274231932126e-05, + "learning_rate": 1.2348413767193759e-05, + "loss": 6.484449954768934e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1848, + "tokens/total": 27902384, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 597428 + }, + { + "epoch": 28.890625, + "grad_norm": 3.446700065978803e-05, + "learning_rate": 1.2325307454490642e-05, + "loss": 7.468387934750353e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1849, + "tokens/total": 27917500, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 597755 + }, + { + "epoch": 28.90625, + "grad_norm": 2.6241561499773525e-05, + "learning_rate": 1.2302312362168934e-05, + "loss": 6.406710326700704e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1850, + "tokens/total": 27932480, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 598073 + }, + { + "epoch": 28.921875, + "grad_norm": 1.3880233382224105e-05, + "learning_rate": 1.2279428550159312e-05, + "loss": 4.768160408730182e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1851, + "tokens/total": 27947692, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 598398 + }, + { + "epoch": 28.9375, + "grad_norm": 7.014289440121502e-05, + "learning_rate": 1.2256656078102408e-05, + "loss": 1.4241528560887673e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1852, + "tokens/total": 27962644, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 598717 + }, + { + "epoch": 28.953125, + "grad_norm": 2.2868682208354585e-05, + "learning_rate": 1.2233995005348711e-05, + "loss": 7.070074161674711e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1853, + "tokens/total": 27977644, + "tokens/train_per_sec_per_gpu": 9.93, + "tokens/trainable": 599036 + }, + { + "epoch": 28.96875, + "grad_norm": 6.826781464042142e-05, + "learning_rate": 1.2211445390958343e-05, + "loss": 9.57587417360628e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1854, + "tokens/total": 27992784, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 599360 + }, + { + "epoch": 28.984375, + "grad_norm": 3.6062214348930866e-05, + "learning_rate": 1.2189007293700957e-05, + "loss": 7.737432952126255e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1855, + "tokens/total": 28007904, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 599686 + }, + { + "epoch": 29.0, + "grad_norm": 3.3886146411532536e-05, + "learning_rate": 1.2166680772055556e-05, + "loss": 9.154649660558789e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1856, + "tokens/total": 28022940, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 600010 + }, + { + "epoch": 29.015625, + "grad_norm": 2.869762829504907e-05, + "learning_rate": 1.2144465884210378e-05, + "loss": 6.9107403533053e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1857, + "tokens/total": 28038132, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 600334 + }, + { + "epoch": 29.03125, + "grad_norm": 6.361609848681837e-05, + "learning_rate": 1.2122362688062671e-05, + "loss": 1.3204405604483327e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1858, + "tokens/total": 28053132, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 600655 + }, + { + "epoch": 29.046875, + "grad_norm": 2.122849036823027e-05, + "learning_rate": 1.2100371241218626e-05, + "loss": 7.096525109773211e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1859, + "tokens/total": 28068220, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 600979 + }, + { + "epoch": 29.0625, + "grad_norm": 1.5158451788011007e-05, + "learning_rate": 1.207849160099317e-05, + "loss": 5.021148581363377e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1860, + "tokens/total": 28083256, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 601302 + }, + { + "epoch": 29.078125, + "grad_norm": 5.677198350895196e-05, + "learning_rate": 1.2056723824409846e-05, + "loss": 1.0023711638496025e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1861, + "tokens/total": 28098408, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 601629 + }, + { + "epoch": 29.09375, + "grad_norm": 3.686436321004294e-05, + "learning_rate": 1.2035067968200637e-05, + "loss": 7.619126449753821e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1862, + "tokens/total": 28113512, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 601949 + }, + { + "epoch": 29.109375, + "grad_norm": 4.002578862127848e-05, + "learning_rate": 1.2013524088805858e-05, + "loss": 7.4802812832786e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1863, + "tokens/total": 28128740, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 602271 + }, + { + "epoch": 29.125, + "grad_norm": 1.6959224012680352e-05, + "learning_rate": 1.199209224237397e-05, + "loss": 5.660208444169257e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1864, + "tokens/total": 28143808, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 602596 + }, + { + "epoch": 29.140625, + "grad_norm": 1.9731947759282775e-05, + "learning_rate": 1.197077248476145e-05, + "loss": 5.852688218510593e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1865, + "tokens/total": 28158936, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 602919 + }, + { + "epoch": 29.15625, + "grad_norm": 0.00014462742547038943, + "learning_rate": 1.1949564871532653e-05, + "loss": 2.226575816166587e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1866, + "tokens/total": 28174140, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 603249 + }, + { + "epoch": 29.171875, + "grad_norm": 1.4596206710848492e-05, + "learning_rate": 1.1928469457959648e-05, + "loss": 4.711623660114128e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1867, + "tokens/total": 28189212, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 603572 + }, + { + "epoch": 29.1875, + "grad_norm": 1.7257822037208825e-05, + "learning_rate": 1.1907486299022106e-05, + "loss": 5.333612875801919e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1868, + "tokens/total": 28204312, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 603898 + }, + { + "epoch": 29.203125, + "grad_norm": 2.0256940842955373e-05, + "learning_rate": 1.1886615449407105e-05, + "loss": 6.474371048170724e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1869, + "tokens/total": 28219420, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 604223 + }, + { + "epoch": 29.21875, + "grad_norm": 3.6564139008987695e-05, + "learning_rate": 1.1865856963509043e-05, + "loss": 9.379527909914032e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1870, + "tokens/total": 28234496, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 604548 + }, + { + "epoch": 29.234375, + "grad_norm": 3.832494985545054e-05, + "learning_rate": 1.1845210895429463e-05, + "loss": 9.517892749499879e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1871, + "tokens/total": 28249660, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 604871 + }, + { + "epoch": 29.25, + "grad_norm": 2.19177090912126e-05, + "learning_rate": 1.1824677298976924e-05, + "loss": 7.475014740521146e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1872, + "tokens/total": 28264744, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 605202 + }, + { + "epoch": 29.265625, + "grad_norm": 1.6569923900533468e-05, + "learning_rate": 1.1804256227666843e-05, + "loss": 5.665755793415883e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1873, + "tokens/total": 28279748, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 605528 + }, + { + "epoch": 29.28125, + "grad_norm": 4.3372441723477095e-05, + "learning_rate": 1.1783947734721387e-05, + "loss": 7.676972018089145e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1874, + "tokens/total": 28294824, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 605846 + }, + { + "epoch": 29.296875, + "grad_norm": 4.2699870391516015e-05, + "learning_rate": 1.1763751873069309e-05, + "loss": 9.350284813081089e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1875, + "tokens/total": 28309928, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 606169 + }, + { + "epoch": 29.3125, + "grad_norm": 2.9506296414183453e-05, + "learning_rate": 1.174366869534583e-05, + "loss": 7.760965559100441e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1876, + "tokens/total": 28325036, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 606493 + }, + { + "epoch": 29.328125, + "grad_norm": 3.064401971641928e-05, + "learning_rate": 1.172369825389246e-05, + "loss": 8.81837991073553e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1877, + "tokens/total": 28340156, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 606812 + }, + { + "epoch": 29.34375, + "grad_norm": 3.552667476469651e-05, + "learning_rate": 1.1703840600756928e-05, + "loss": 8.680675591676845e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1878, + "tokens/total": 28355312, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 607131 + }, + { + "epoch": 29.359375, + "grad_norm": 1.826221887313295e-05, + "learning_rate": 1.1684095787692985e-05, + "loss": 6.303241661953507e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1879, + "tokens/total": 28370432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 607452 + }, + { + "epoch": 29.375, + "grad_norm": 1.8276723494636826e-05, + "learning_rate": 1.1664463866160308e-05, + "loss": 5.879717832613096e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1880, + "tokens/total": 28385528, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 607773 + }, + { + "epoch": 29.390625, + "grad_norm": 2.2807898858445697e-05, + "learning_rate": 1.1644944887324339e-05, + "loss": 6.598353934350598e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1881, + "tokens/total": 28400768, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 608099 + }, + { + "epoch": 29.40625, + "grad_norm": 1.952206548594404e-05, + "learning_rate": 1.1625538902056175e-05, + "loss": 5.94521225139033e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1882, + "tokens/total": 28415948, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 608423 + }, + { + "epoch": 29.421875, + "grad_norm": 1.9541654182830825e-05, + "learning_rate": 1.1606245960932421e-05, + "loss": 6.739164746250026e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1883, + "tokens/total": 28431112, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 608745 + }, + { + "epoch": 29.4375, + "grad_norm": 4.003664435003884e-05, + "learning_rate": 1.1587066114235063e-05, + "loss": 8.235517157118011e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1884, + "tokens/total": 28446276, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 609068 + }, + { + "epoch": 29.453125, + "grad_norm": 2.0218769350321963e-05, + "learning_rate": 1.1567999411951337e-05, + "loss": 6.50548315661581e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1885, + "tokens/total": 28461448, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 609397 + }, + { + "epoch": 29.46875, + "grad_norm": 2.0576882889145054e-05, + "learning_rate": 1.154904590377359e-05, + "loss": 7.390931955342239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1886, + "tokens/total": 28476492, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 609723 + }, + { + "epoch": 29.484375, + "grad_norm": 2.089896770485211e-05, + "learning_rate": 1.1530205639099166e-05, + "loss": 6.519133535221044e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1887, + "tokens/total": 28491620, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 610046 + }, + { + "epoch": 29.5, + "grad_norm": 2.18697950913338e-05, + "learning_rate": 1.151147866703028e-05, + "loss": 7.214358674900723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1888, + "tokens/total": 28506804, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 610368 + }, + { + "epoch": 29.515625, + "grad_norm": 2.7423302526585758e-05, + "learning_rate": 1.1492865036373855e-05, + "loss": 7.454198680534319e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1889, + "tokens/total": 28521888, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 610696 + }, + { + "epoch": 29.53125, + "grad_norm": 1.9132477973471396e-05, + "learning_rate": 1.1474364795641428e-05, + "loss": 5.997104608468362e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1890, + "tokens/total": 28537140, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 611017 + }, + { + "epoch": 29.546875, + "grad_norm": 1.8364531570114195e-05, + "learning_rate": 1.1455977993049036e-05, + "loss": 5.331530132934859e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1891, + "tokens/total": 28552232, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 611339 + }, + { + "epoch": 29.5625, + "grad_norm": 2.921898521890398e-05, + "learning_rate": 1.1437704676517041e-05, + "loss": 7.226759066725208e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1892, + "tokens/total": 28567348, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 611662 + }, + { + "epoch": 29.578125, + "grad_norm": 2.6524958229856566e-05, + "learning_rate": 1.141954489367005e-05, + "loss": 7.622857083333656e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1893, + "tokens/total": 28582492, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 611986 + }, + { + "epoch": 29.59375, + "grad_norm": 1.4574326087313239e-05, + "learning_rate": 1.1401498691836757e-05, + "loss": 4.750260700348008e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1894, + "tokens/total": 28597584, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 612315 + }, + { + "epoch": 29.609375, + "grad_norm": 2.4668675905559212e-05, + "learning_rate": 1.1383566118049863e-05, + "loss": 7.825965440133587e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1895, + "tokens/total": 28612636, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 612639 + }, + { + "epoch": 29.625, + "grad_norm": 2.2862521291244775e-05, + "learning_rate": 1.1365747219045907e-05, + "loss": 6.835816179773246e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1896, + "tokens/total": 28627708, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 612965 + }, + { + "epoch": 29.640625, + "grad_norm": 2.4580529498052783e-05, + "learning_rate": 1.1348042041265174e-05, + "loss": 7.765044642837893e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1897, + "tokens/total": 28642880, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 613291 + }, + { + "epoch": 29.65625, + "grad_norm": 6.363989086821675e-05, + "learning_rate": 1.1330450630851554e-05, + "loss": 8.31935835776676e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1898, + "tokens/total": 28658004, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 613613 + }, + { + "epoch": 29.671875, + "grad_norm": 1.9392153262742795e-05, + "learning_rate": 1.1312973033652449e-05, + "loss": 5.838567176397191e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1899, + "tokens/total": 28673044, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 613936 + }, + { + "epoch": 29.6875, + "grad_norm": 1.9203220290364698e-05, + "learning_rate": 1.1295609295218621e-05, + "loss": 5.708837989004678e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1900, + "tokens/total": 28688188, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 614257 + }, + { + "epoch": 29.703125, + "grad_norm": 1.9975586837972514e-05, + "learning_rate": 1.1278359460804097e-05, + "loss": 6.900319817759737e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1901, + "tokens/total": 28703332, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 614578 + }, + { + "epoch": 29.71875, + "grad_norm": 5.7108358305413276e-05, + "learning_rate": 1.126122357536604e-05, + "loss": 1.197800202135113e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1902, + "tokens/total": 28718344, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 614900 + }, + { + "epoch": 29.734375, + "grad_norm": 2.2429978344007395e-05, + "learning_rate": 1.124420168356463e-05, + "loss": 7.206799637060612e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1903, + "tokens/total": 28733420, + "tokens/train_per_sec_per_gpu": 8.92, + "tokens/trainable": 615224 + }, + { + "epoch": 29.75, + "grad_norm": 1.8767070287140086e-05, + "learning_rate": 1.1227293829762976e-05, + "loss": 5.717983526665193e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1904, + "tokens/total": 28748388, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 615549 + }, + { + "epoch": 29.765625, + "grad_norm": 2.7467664040159434e-05, + "learning_rate": 1.1210500058026946e-05, + "loss": 7.219958320092701e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1905, + "tokens/total": 28763480, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 615868 + }, + { + "epoch": 29.78125, + "grad_norm": 2.301012682437431e-05, + "learning_rate": 1.11938204121251e-05, + "loss": 7.472759762094938e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1906, + "tokens/total": 28778696, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 616188 + }, + { + "epoch": 29.796875, + "grad_norm": 2.1394911527750082e-05, + "learning_rate": 1.1177254935528556e-05, + "loss": 6.772941105737118e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1907, + "tokens/total": 28793744, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 616507 + }, + { + "epoch": 29.8125, + "grad_norm": 1.4847312741039786e-05, + "learning_rate": 1.1160803671410877e-05, + "loss": 4.809053280041553e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1908, + "tokens/total": 28808884, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 616828 + }, + { + "epoch": 29.828125, + "grad_norm": 1.6719141058274545e-05, + "learning_rate": 1.1144466662647961e-05, + "loss": 5.369127507037774e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1909, + "tokens/total": 28823968, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 617148 + }, + { + "epoch": 29.84375, + "grad_norm": 2.6284595151082613e-05, + "learning_rate": 1.1128243951817937e-05, + "loss": 8.694694315636298e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1910, + "tokens/total": 28838976, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 617471 + }, + { + "epoch": 29.859375, + "grad_norm": 6.230138387763873e-05, + "learning_rate": 1.1112135581201048e-05, + "loss": 9.786887176232995e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1911, + "tokens/total": 28853956, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 617793 + }, + { + "epoch": 29.875, + "grad_norm": 2.421082172077149e-05, + "learning_rate": 1.1096141592779522e-05, + "loss": 5.204944955039537e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1912, + "tokens/total": 28869048, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 618120 + }, + { + "epoch": 29.890625, + "grad_norm": 3.786037632380612e-05, + "learning_rate": 1.1080262028237489e-05, + "loss": 8.70413941811421e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1913, + "tokens/total": 28884128, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 618440 + }, + { + "epoch": 29.90625, + "grad_norm": 3.316303627798334e-05, + "learning_rate": 1.1064496928960872e-05, + "loss": 8.012565331227961e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1914, + "tokens/total": 28899212, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 618766 + }, + { + "epoch": 29.921875, + "grad_norm": 2.6983805582858622e-05, + "learning_rate": 1.104884633603726e-05, + "loss": 8.085100944299484e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1915, + "tokens/total": 28914220, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 619085 + }, + { + "epoch": 29.9375, + "grad_norm": 2.870435309887398e-05, + "learning_rate": 1.1033310290255813e-05, + "loss": 8.236646635850775e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1916, + "tokens/total": 28929316, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 619408 + }, + { + "epoch": 29.953125, + "grad_norm": 1.8335444110562094e-05, + "learning_rate": 1.1017888832107153e-05, + "loss": 6.037054731677927e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1917, + "tokens/total": 28944400, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 619731 + }, + { + "epoch": 29.96875, + "grad_norm": 7.943164382595569e-05, + "learning_rate": 1.1002582001783273e-05, + "loss": 9.540684686726308e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1918, + "tokens/total": 28959544, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 620055 + }, + { + "epoch": 29.984375, + "grad_norm": 3.363615178386681e-05, + "learning_rate": 1.0987389839177397e-05, + "loss": 8.251199687947519e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1919, + "tokens/total": 28974588, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 620377 + }, + { + "epoch": 30.0, + "grad_norm": 1.954712570295669e-05, + "learning_rate": 1.097231238388391e-05, + "loss": 6.40598898371536e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1920, + "tokens/total": 28989656, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 620700 + }, + { + "epoch": 30.015625, + "grad_norm": 3.0150709790177643e-05, + "learning_rate": 1.0957349675198241e-05, + "loss": 7.088405027388944e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1921, + "tokens/total": 29004788, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 621021 + }, + { + "epoch": 30.03125, + "grad_norm": 3.597562317736447e-05, + "learning_rate": 1.0942501752116758e-05, + "loss": 8.399876492148906e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1922, + "tokens/total": 29019884, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 621338 + }, + { + "epoch": 30.046875, + "grad_norm": 2.1559850210906006e-05, + "learning_rate": 1.0927768653336678e-05, + "loss": 6.461492603193619e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1923, + "tokens/total": 29035048, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 621665 + }, + { + "epoch": 30.0625, + "grad_norm": 2.4845145162544213e-05, + "learning_rate": 1.0913150417255957e-05, + "loss": 8.339582109329058e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1924, + "tokens/total": 29050072, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 621986 + }, + { + "epoch": 30.078125, + "grad_norm": 1.9045726730837487e-05, + "learning_rate": 1.0898647081973187e-05, + "loss": 6.285619065238279e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1925, + "tokens/total": 29065208, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 622308 + }, + { + "epoch": 30.09375, + "grad_norm": 7.231788185890764e-05, + "learning_rate": 1.0884258685287498e-05, + "loss": 5.659799739987648e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1926, + "tokens/total": 29080328, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 622630 + }, + { + "epoch": 30.109375, + "grad_norm": 1.736078593239654e-05, + "learning_rate": 1.0869985264698477e-05, + "loss": 5.731166652367392e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1927, + "tokens/total": 29095452, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 622957 + }, + { + "epoch": 30.125, + "grad_norm": 1.4173566341924015e-05, + "learning_rate": 1.0855826857406036e-05, + "loss": 4.801988211511343e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1928, + "tokens/total": 29110532, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 623281 + }, + { + "epoch": 30.140625, + "grad_norm": 5.4095475206850097e-05, + "learning_rate": 1.0841783500310361e-05, + "loss": 9.777942295841058e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1929, + "tokens/total": 29125644, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 623601 + }, + { + "epoch": 30.15625, + "grad_norm": 1.6718735423637554e-05, + "learning_rate": 1.0827855230011758e-05, + "loss": 3.6808035019930685e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1930, + "tokens/total": 29140640, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 623924 + }, + { + "epoch": 30.171875, + "grad_norm": 2.153054447262548e-05, + "learning_rate": 1.0814042082810625e-05, + "loss": 6.388726205841522e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1931, + "tokens/total": 29155720, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 624247 + }, + { + "epoch": 30.1875, + "grad_norm": 1.8344408090342768e-05, + "learning_rate": 1.0800344094707283e-05, + "loss": 6.297649406405981e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1932, + "tokens/total": 29170876, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 624571 + }, + { + "epoch": 30.203125, + "grad_norm": 1.767187495715916e-05, + "learning_rate": 1.0786761301401964e-05, + "loss": 6.028819257153373e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1933, + "tokens/total": 29186040, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 624891 + }, + { + "epoch": 30.21875, + "grad_norm": 2.0285493519622833e-05, + "learning_rate": 1.0773293738294627e-05, + "loss": 7.054614457047137e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1934, + "tokens/total": 29201004, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 625212 + }, + { + "epoch": 30.234375, + "grad_norm": 2.7763644538936205e-05, + "learning_rate": 1.0759941440484967e-05, + "loss": 9.017629736263189e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1935, + "tokens/total": 29216160, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 625535 + }, + { + "epoch": 30.25, + "grad_norm": 1.8593649656395428e-05, + "learning_rate": 1.0746704442772222e-05, + "loss": 6.101570306782378e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1936, + "tokens/total": 29231316, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 625855 + }, + { + "epoch": 30.265625, + "grad_norm": 2.716101153055206e-05, + "learning_rate": 1.0733582779655157e-05, + "loss": 7.743926744296914e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1937, + "tokens/total": 29246604, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 626176 + }, + { + "epoch": 30.28125, + "grad_norm": 1.2336143299762625e-05, + "learning_rate": 1.0720576485331943e-05, + "loss": 4.1834002217910893e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1938, + "tokens/total": 29261712, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 626501 + }, + { + "epoch": 30.296875, + "grad_norm": 2.4514292817912064e-05, + "learning_rate": 1.0707685593700068e-05, + "loss": 7.636936061317101e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1939, + "tokens/total": 29276760, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 626826 + }, + { + "epoch": 30.3125, + "grad_norm": 1.989788688661065e-05, + "learning_rate": 1.0694910138356263e-05, + "loss": 6.528692892970867e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1940, + "tokens/total": 29291932, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 627149 + }, + { + "epoch": 30.328125, + "grad_norm": 2.6206509573967196e-05, + "learning_rate": 1.068225015259639e-05, + "loss": 7.254164984260569e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1941, + "tokens/total": 29307036, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 627476 + }, + { + "epoch": 30.34375, + "grad_norm": 2.354349817323964e-05, + "learning_rate": 1.0669705669415384e-05, + "loss": 6.852218916719721e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1942, + "tokens/total": 29322052, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 627796 + }, + { + "epoch": 30.359375, + "grad_norm": 3.135994847980328e-05, + "learning_rate": 1.0657276721507142e-05, + "loss": 8.640311079943785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1943, + "tokens/total": 29337140, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 628117 + }, + { + "epoch": 30.375, + "grad_norm": 6.997265154495835e-05, + "learning_rate": 1.0644963341264458e-05, + "loss": 9.359050636703614e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1944, + "tokens/total": 29352196, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 628443 + }, + { + "epoch": 30.390625, + "grad_norm": 7.97888424131088e-05, + "learning_rate": 1.0632765560778915e-05, + "loss": 7.055046467030479e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1945, + "tokens/total": 29367204, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 628773 + }, + { + "epoch": 30.40625, + "grad_norm": 5.649407466989942e-05, + "learning_rate": 1.0620683411840844e-05, + "loss": 5.591786020886502e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1946, + "tokens/total": 29382268, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 629101 + }, + { + "epoch": 30.421875, + "grad_norm": 1.9991593944723718e-05, + "learning_rate": 1.0608716925939172e-05, + "loss": 6.75553110340843e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1947, + "tokens/total": 29397348, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 629424 + }, + { + "epoch": 30.4375, + "grad_norm": 4.544229159364477e-05, + "learning_rate": 1.0596866134261421e-05, + "loss": 8.308697942993604e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1948, + "tokens/total": 29412404, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 629749 + }, + { + "epoch": 30.453125, + "grad_norm": 1.6721316569601186e-05, + "learning_rate": 1.058513106769355e-05, + "loss": 5.375025011744583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1949, + "tokens/total": 29427336, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 630071 + }, + { + "epoch": 30.46875, + "grad_norm": 2.384197068749927e-05, + "learning_rate": 1.0573511756819949e-05, + "loss": 6.652340971413651e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1950, + "tokens/total": 29442544, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 630395 + }, + { + "epoch": 30.484375, + "grad_norm": 5.8081059250980616e-05, + "learning_rate": 1.0562008231923284e-05, + "loss": 1.0572821338428184e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1951, + "tokens/total": 29457600, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 630719 + }, + { + "epoch": 30.5, + "grad_norm": 1.878198963822797e-05, + "learning_rate": 1.0550620522984483e-05, + "loss": 5.881781817151932e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1952, + "tokens/total": 29472684, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 631041 + }, + { + "epoch": 30.515625, + "grad_norm": 1.8786731743603013e-05, + "learning_rate": 1.053934865968261e-05, + "loss": 5.699433813788346e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1953, + "tokens/total": 29487708, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 631365 + }, + { + "epoch": 30.53125, + "grad_norm": 1.5066514606587589e-05, + "learning_rate": 1.0528192671394826e-05, + "loss": 5.196815209274064e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1954, + "tokens/total": 29502820, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 631688 + }, + { + "epoch": 30.546875, + "grad_norm": 1.5496141713811085e-05, + "learning_rate": 1.0517152587196279e-05, + "loss": 5.255628252598399e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1955, + "tokens/total": 29518012, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 632009 + }, + { + "epoch": 30.5625, + "grad_norm": 3.0141169190756045e-05, + "learning_rate": 1.0506228435860055e-05, + "loss": 7.335980853895308e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1956, + "tokens/total": 29533048, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 632328 + }, + { + "epoch": 30.578125, + "grad_norm": 1.7384680177201517e-05, + "learning_rate": 1.0495420245857086e-05, + "loss": 5.563425702348468e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1957, + "tokens/total": 29548156, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 632658 + }, + { + "epoch": 30.59375, + "grad_norm": 4.2620638851076365e-05, + "learning_rate": 1.0484728045356076e-05, + "loss": 1.0345362397856661e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1958, + "tokens/total": 29563248, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 632982 + }, + { + "epoch": 30.609375, + "grad_norm": 2.2986152544035576e-05, + "learning_rate": 1.0474151862223446e-05, + "loss": 7.970745627972065e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1959, + "tokens/total": 29578392, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 633305 + }, + { + "epoch": 30.625, + "grad_norm": 3.7342724681366235e-05, + "learning_rate": 1.0463691724023228e-05, + "loss": 7.994262887223158e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1960, + "tokens/total": 29593432, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 633630 + }, + { + "epoch": 30.640625, + "grad_norm": 2.4177612431230955e-05, + "learning_rate": 1.045334765801704e-05, + "loss": 7.480267640858074e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1961, + "tokens/total": 29608492, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 633953 + }, + { + "epoch": 30.65625, + "grad_norm": 1.7861308151623234e-05, + "learning_rate": 1.0443119691163966e-05, + "loss": 5.897734354221029e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1962, + "tokens/total": 29623732, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 634277 + }, + { + "epoch": 30.671875, + "grad_norm": 2.0189436327200383e-05, + "learning_rate": 1.0433007850120519e-05, + "loss": 6.599445327992726e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1963, + "tokens/total": 29638924, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 634597 + }, + { + "epoch": 30.6875, + "grad_norm": 5.839717778144404e-05, + "learning_rate": 1.042301216124055e-05, + "loss": 5.434261538539431e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1964, + "tokens/total": 29654004, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 634919 + }, + { + "epoch": 30.703125, + "grad_norm": 1.3869013855583034e-05, + "learning_rate": 1.0413132650575213e-05, + "loss": 4.3461363929964136e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1965, + "tokens/total": 29669244, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 635245 + }, + { + "epoch": 30.71875, + "grad_norm": 1.5403742509079166e-05, + "learning_rate": 1.0403369343872845e-05, + "loss": 5.203896193961555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1966, + "tokens/total": 29684384, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 635568 + }, + { + "epoch": 30.734375, + "grad_norm": 5.6457891332684085e-05, + "learning_rate": 1.0393722266578948e-05, + "loss": 1.1889396773767658e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1967, + "tokens/total": 29699476, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 635893 + }, + { + "epoch": 30.75, + "grad_norm": 2.312154720129911e-05, + "learning_rate": 1.0384191443836093e-05, + "loss": 7.327433877435396e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1968, + "tokens/total": 29714576, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 636215 + }, + { + "epoch": 30.765625, + "grad_norm": 5.5283708206843585e-05, + "learning_rate": 1.0374776900483867e-05, + "loss": 9.504086619926966e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1969, + "tokens/total": 29729600, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 636535 + }, + { + "epoch": 30.78125, + "grad_norm": 1.8025279132416472e-05, + "learning_rate": 1.036547866105881e-05, + "loss": 5.694143965229159e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1970, + "tokens/total": 29744708, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 636858 + }, + { + "epoch": 30.796875, + "grad_norm": 2.0646362827392295e-05, + "learning_rate": 1.0356296749794338e-05, + "loss": 6.801454333071888e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1971, + "tokens/total": 29759712, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 637183 + }, + { + "epoch": 30.8125, + "grad_norm": 3.637433110270649e-05, + "learning_rate": 1.0347231190620688e-05, + "loss": 4.831669571103703e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1972, + "tokens/total": 29774784, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 637504 + }, + { + "epoch": 30.828125, + "grad_norm": 0.00013172195758670568, + "learning_rate": 1.0338282007164868e-05, + "loss": 2.353975332880509e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1973, + "tokens/total": 29789852, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 637826 + }, + { + "epoch": 30.84375, + "grad_norm": 1.7440581359551288e-05, + "learning_rate": 1.0329449222750567e-05, + "loss": 5.063664616500319e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1974, + "tokens/total": 29804892, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 638150 + }, + { + "epoch": 30.859375, + "grad_norm": 1.5657969925086945e-05, + "learning_rate": 1.0320732860398126e-05, + "loss": 5.160578098184487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1975, + "tokens/total": 29820144, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 638476 + }, + { + "epoch": 30.875, + "grad_norm": 0.00018536881543695927, + "learning_rate": 1.031213294282445e-05, + "loss": 9.09727987163933e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1976, + "tokens/total": 29835260, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 638799 + }, + { + "epoch": 30.890625, + "grad_norm": 2.5382210878888145e-05, + "learning_rate": 1.0303649492442971e-05, + "loss": 6.757573487448099e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1977, + "tokens/total": 29850364, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 639121 + }, + { + "epoch": 30.90625, + "grad_norm": 1.7607142581255175e-05, + "learning_rate": 1.029528253136357e-05, + "loss": 6.16681234077987e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1978, + "tokens/total": 29865496, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 639446 + }, + { + "epoch": 30.921875, + "grad_norm": 1.7890299204736948e-05, + "learning_rate": 1.028703208139253e-05, + "loss": 5.811500614072429e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1979, + "tokens/total": 29880492, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 639772 + }, + { + "epoch": 30.9375, + "grad_norm": 1.9137485651299357e-05, + "learning_rate": 1.0278898164032491e-05, + "loss": 6.493069122370798e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1980, + "tokens/total": 29895648, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 640098 + }, + { + "epoch": 30.953125, + "grad_norm": 2.1833826394868083e-05, + "learning_rate": 1.0270880800482372e-05, + "loss": 7.442793616974086e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1981, + "tokens/total": 29910740, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 640424 + }, + { + "epoch": 30.96875, + "grad_norm": 5.4120708227856085e-05, + "learning_rate": 1.0262980011637325e-05, + "loss": 1.2726327440759633e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1982, + "tokens/total": 29925904, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 640745 + }, + { + "epoch": 30.984375, + "grad_norm": 3.8585319998674095e-05, + "learning_rate": 1.0255195818088677e-05, + "loss": 8.244147124969459e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1983, + "tokens/total": 29941016, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 641070 + }, + { + "epoch": 31.0, + "grad_norm": 3.817715696641244e-05, + "learning_rate": 1.0247528240123894e-05, + "loss": 8.85582039700239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.0, + "step": 1984, + "tokens/total": 29955968, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 641390 + }, + { + "epoch": 31.015625, + "grad_norm": 8.151040674420074e-05, + "learning_rate": 1.0239977297726498e-05, + "loss": 8.160709512594622e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1985, + "tokens/total": 29971064, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 641717 + }, + { + "epoch": 31.03125, + "grad_norm": 1.917262670758646e-05, + "learning_rate": 1.0232543010576036e-05, + "loss": 6.105462944105966e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1986, + "tokens/total": 29986064, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 642035 + }, + { + "epoch": 31.046875, + "grad_norm": 3.748734889086336e-05, + "learning_rate": 1.0225225398048028e-05, + "loss": 6.950261308702466e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1987, + "tokens/total": 30001116, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 642358 + }, + { + "epoch": 31.0625, + "grad_norm": 1.7289201423409395e-05, + "learning_rate": 1.0218024479213911e-05, + "loss": 5.875382385056582e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1988, + "tokens/total": 30016220, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 642677 + }, + { + "epoch": 31.078125, + "grad_norm": 1.7349420886603184e-05, + "learning_rate": 1.0210940272840991e-05, + "loss": 5.891815817449242e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1989, + "tokens/total": 30031380, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 643003 + }, + { + "epoch": 31.09375, + "grad_norm": 2.9556615118053742e-05, + "learning_rate": 1.0203972797392391e-05, + "loss": 8.722475968170329e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1990, + "tokens/total": 30046396, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 643327 + }, + { + "epoch": 31.109375, + "grad_norm": 2.2310134227154776e-05, + "learning_rate": 1.0197122071026999e-05, + "loss": 6.534580734296469e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1991, + "tokens/total": 30061396, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643648 + }, + { + "epoch": 31.125, + "grad_norm": 3.6734334571519867e-05, + "learning_rate": 1.0190388111599446e-05, + "loss": 8.670037345837045e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1992, + "tokens/total": 30076380, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 643967 + }, + { + "epoch": 31.140625, + "grad_norm": 3.7599886127281934e-05, + "learning_rate": 1.0183770936660021e-05, + "loss": 8.707866641088913e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1993, + "tokens/total": 30091484, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 644290 + }, + { + "epoch": 31.15625, + "grad_norm": 1.873763721960131e-05, + "learning_rate": 1.0177270563454658e-05, + "loss": 5.571010888161254e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1994, + "tokens/total": 30106668, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 644614 + }, + { + "epoch": 31.171875, + "grad_norm": 2.1826010197401047e-05, + "learning_rate": 1.017088700892487e-05, + "loss": 6.015002895765065e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1995, + "tokens/total": 30121836, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 644934 + }, + { + "epoch": 31.1875, + "grad_norm": 1.6165675333468243e-05, + "learning_rate": 1.0164620289707715e-05, + "loss": 5.584131486102706e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1996, + "tokens/total": 30136764, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 645256 + }, + { + "epoch": 31.203125, + "grad_norm": 1.587179212947376e-05, + "learning_rate": 1.0158470422135751e-05, + "loss": 4.855938868786325e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1997, + "tokens/total": 30151912, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 645582 + }, + { + "epoch": 31.21875, + "grad_norm": 0.00024376699002459645, + "learning_rate": 1.0152437422236999e-05, + "loss": 9.44419980442035e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1998, + "tokens/total": 30166936, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 645907 + }, + { + "epoch": 31.234375, + "grad_norm": 2.8756183382938616e-05, + "learning_rate": 1.0146521305734883e-05, + "loss": 8.242810167757852e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1999, + "tokens/total": 30181996, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 646237 + }, + { + "epoch": 31.25, + "grad_norm": 5.1261224143672734e-05, + "learning_rate": 1.0140722088048205e-05, + "loss": 8.611204407316109e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2000, + "tokens/total": 30197008, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 646560 + }, + { + "epoch": 31.265625, + "grad_norm": 2.3335760488407686e-05, + "learning_rate": 1.0135039784291113e-05, + "loss": 7.26922166904842e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2001, + "tokens/total": 30212100, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 646884 + }, + { + "epoch": 31.28125, + "grad_norm": 2.6369583792984486e-05, + "learning_rate": 1.0129474409273033e-05, + "loss": 7.947530775709311e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2002, + "tokens/total": 30227228, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 647204 + }, + { + "epoch": 31.296875, + "grad_norm": 0.00013051781570538878, + "learning_rate": 1.0124025977498657e-05, + "loss": 1.9139665710099507e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 2003, + "tokens/total": 30242208, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 647527 + }, + { + "epoch": 31.3125, + "grad_norm": 1.6326494005625136e-05, + "learning_rate": 1.0118694503167885e-05, + "loss": 5.315732209965063e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 2004, + "tokens/total": 30257356, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 647852 + }, + { + "epoch": 31.328125, + "grad_norm": 2.083092840621248e-05, + "learning_rate": 1.0113480000175818e-05, + "loss": 6.6166364831588e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2005, + "tokens/total": 30272452, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 648176 + }, + { + "epoch": 31.34375, + "grad_norm": 1.995458296732977e-05, + "learning_rate": 1.0108382482112672e-05, + "loss": 6.69668736463791e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2006, + "tokens/total": 30287584, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 648497 + }, + { + "epoch": 31.359375, + "grad_norm": 1.714762947813142e-05, + "learning_rate": 1.0103401962263806e-05, + "loss": 5.289797400109819e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2007, + "tokens/total": 30302644, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 648816 + }, + { + "epoch": 31.375, + "grad_norm": 2.0432820747373626e-05, + "learning_rate": 1.0098538453609624e-05, + "loss": 7.014650691417046e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2008, + "tokens/total": 30317716, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 649140 + }, + { + "epoch": 31.390625, + "grad_norm": 1.6683699868735857e-05, + "learning_rate": 1.009379196882559e-05, + "loss": 5.453939024846477e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 2009, + "tokens/total": 30332832, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 649464 + }, + { + "epoch": 31.40625, + "grad_norm": 5.379769572755322e-05, + "learning_rate": 1.0089162520282172e-05, + "loss": 1.025085111905355e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2010, + "tokens/total": 30347944, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 649784 + }, + { + "epoch": 31.421875, + "grad_norm": 1.7693910194793716e-05, + "learning_rate": 1.0084650120044816e-05, + "loss": 5.943283554188383e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2011, + "tokens/total": 30363124, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 650108 + }, + { + "epoch": 31.4375, + "grad_norm": 1.4680968888569623e-05, + "learning_rate": 1.0080254779873893e-05, + "loss": 4.6087930627436435e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2012, + "tokens/total": 30378296, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 650433 + }, + { + "epoch": 31.453125, + "grad_norm": 1.503605381003581e-05, + "learning_rate": 1.0075976511224718e-05, + "loss": 5.14154237407638e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2013, + "tokens/total": 30393396, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 650759 + }, + { + "epoch": 31.46875, + "grad_norm": 2.1122383259353228e-05, + "learning_rate": 1.0071815325247474e-05, + "loss": 6.922582542756572e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 2014, + "tokens/total": 30408500, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 651078 + }, + { + "epoch": 31.484375, + "grad_norm": 1.6383302863687277e-05, + "learning_rate": 1.006777123278719e-05, + "loss": 5.664404625349562e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2015, + "tokens/total": 30423568, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 651399 + }, + { + "epoch": 31.5, + "grad_norm": 3.546892185113393e-05, + "learning_rate": 1.0063844244383736e-05, + "loss": 8.43410532525013e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2016, + "tokens/total": 30438648, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 651720 + }, + { + "epoch": 31.515625, + "grad_norm": 3.174735320499167e-05, + "learning_rate": 1.006003437027178e-05, + "loss": 4.900810495200858e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2017, + "tokens/total": 30453668, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 652046 + }, + { + "epoch": 31.53125, + "grad_norm": 1.9111095753032714e-05, + "learning_rate": 1.0056341620380752e-05, + "loss": 5.811197638649901e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2018, + "tokens/total": 30468808, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 652370 + }, + { + "epoch": 31.546875, + "grad_norm": 1.4540622032654937e-05, + "learning_rate": 1.0052766004334845e-05, + "loss": 4.611223118899943e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 2019, + "tokens/total": 30483776, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 652694 + }, + { + "epoch": 31.5625, + "grad_norm": 2.3490165403927676e-05, + "learning_rate": 1.004930753145295e-05, + "loss": 7.591340249746281e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2020, + "tokens/total": 30498864, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 653019 + }, + { + "epoch": 31.578125, + "grad_norm": 3.217229459551163e-05, + "learning_rate": 1.004596621074868e-05, + "loss": 7.861335689085536e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2021, + "tokens/total": 30514044, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 653341 + }, + { + "epoch": 31.59375, + "grad_norm": 1.9949162378907204e-05, + "learning_rate": 1.0042742050930303e-05, + "loss": 6.590954626517487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2022, + "tokens/total": 30529208, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 653661 + }, + { + "epoch": 31.609375, + "grad_norm": 1.588710983924102e-05, + "learning_rate": 1.0039635060400747e-05, + "loss": 5.216129466134589e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 2023, + "tokens/total": 30544296, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 653983 + }, + { + "epoch": 31.625, + "grad_norm": 3.447076596785337e-05, + "learning_rate": 1.0036645247257572e-05, + "loss": 8.882461202119885e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2024, + "tokens/total": 30559248, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 654306 + }, + { + "epoch": 31.640625, + "grad_norm": 1.3358136129681952e-05, + "learning_rate": 1.0033772619292936e-05, + "loss": 4.5821522576261486e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2025, + "tokens/total": 30574408, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 654634 + }, + { + "epoch": 31.65625, + "grad_norm": 1.7366048268740997e-05, + "learning_rate": 1.0031017183993589e-05, + "loss": 5.657474275722052e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 2026, + "tokens/total": 30589532, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 654963 + }, + { + "epoch": 31.671875, + "grad_norm": 1.7844820831669495e-05, + "learning_rate": 1.0028378948540847e-05, + "loss": 5.838537617819384e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2027, + "tokens/total": 30604664, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 655285 + }, + { + "epoch": 31.6875, + "grad_norm": 2.1518018911592662e-05, + "learning_rate": 1.0025857919810583e-05, + "loss": 6.492567763416446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2028, + "tokens/total": 30619804, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 655610 + }, + { + "epoch": 31.703125, + "grad_norm": 2.036645673797466e-05, + "learning_rate": 1.0023454104373196e-05, + "loss": 6.278289106376178e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2029, + "tokens/total": 30634832, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 655938 + }, + { + "epoch": 31.71875, + "grad_norm": 1.691148463578429e-05, + "learning_rate": 1.0021167508493605e-05, + "loss": 5.091551997793431e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2030, + "tokens/total": 30650048, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 656261 + }, + { + "epoch": 31.734375, + "grad_norm": 1.996784340008162e-05, + "learning_rate": 1.0018998138131207e-05, + "loss": 6.261908538363059e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 2031, + "tokens/total": 30665232, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 656581 + }, + { + "epoch": 31.75, + "grad_norm": 5.151877485332079e-05, + "learning_rate": 1.0016945998939908e-05, + "loss": 1.0669130006135674e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2032, + "tokens/total": 30680472, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 656908 + }, + { + "epoch": 31.765625, + "grad_norm": 1.759117549227085e-05, + "learning_rate": 1.001501109626807e-05, + "loss": 5.579971684710472e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2033, + "tokens/total": 30695508, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 657230 + }, + { + "epoch": 31.78125, + "grad_norm": 1.1838836144306697e-05, + "learning_rate": 1.0013193435158503e-05, + "loss": 3.7526598362092045e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 2034, + "tokens/total": 30710592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 657554 + }, + { + "epoch": 31.796875, + "grad_norm": 1.7554493751958944e-05, + "learning_rate": 1.001149302034847e-05, + "loss": 6.047995384506066e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 2035, + "tokens/total": 30725724, + "tokens/train_per_sec_per_gpu": 10.29, + "tokens/trainable": 657876 + }, + { + "epoch": 31.8125, + "grad_norm": 2.3976539523573592e-05, + "learning_rate": 1.0009909856269647e-05, + "loss": 7.44873375424504e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 2036, + "tokens/total": 30740820, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 658192 + }, + { + "epoch": 31.828125, + "grad_norm": 3.543664934113622e-05, + "learning_rate": 1.0008443947048141e-05, + "loss": 8.302781679958571e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2037, + "tokens/total": 30755904, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 658516 + }, + { + "epoch": 31.84375, + "grad_norm": 7.215787627501413e-05, + "learning_rate": 1.0007095296504465e-05, + "loss": 1.4569216091331327e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2038, + "tokens/total": 30771040, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 658838 + }, + { + "epoch": 31.859375, + "grad_norm": 2.3879636501078494e-05, + "learning_rate": 1.0005863908153509e-05, + "loss": 7.728438049525721e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2039, + "tokens/total": 30786180, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 659156 + }, + { + "epoch": 31.875, + "grad_norm": 2.0198131096549332e-05, + "learning_rate": 1.0004749785204575e-05, + "loss": 6.647691179750836e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2040, + "tokens/total": 30801260, + "tokens/train_per_sec_per_gpu": 8.92, + "tokens/trainable": 659479 + }, + { + "epoch": 31.890625, + "grad_norm": 1.9014716599485837e-05, + "learning_rate": 1.0003752930561329e-05, + "loss": 6.254734330468636e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2041, + "tokens/total": 30816280, + "tokens/train_per_sec_per_gpu": 8.73, + "tokens/trainable": 659804 + }, + { + "epoch": 31.90625, + "grad_norm": 2.0192032025079243e-05, + "learning_rate": 1.0002873346821806e-05, + "loss": 6.968926413719601e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2042, + "tokens/total": 30831464, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 660133 + }, + { + "epoch": 31.921875, + "grad_norm": 3.491413372103125e-05, + "learning_rate": 1.0002111036278416e-05, + "loss": 6.642127914346929e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2043, + "tokens/total": 30846600, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 660458 + }, + { + "epoch": 31.9375, + "grad_norm": 6.281802052399144e-05, + "learning_rate": 1.000146600091792e-05, + "loss": 1.0414059943286702e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2044, + "tokens/total": 30861668, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 660781 + }, + { + "epoch": 31.953125, + "grad_norm": 2.2359387003234588e-05, + "learning_rate": 1.0000938242421431e-05, + "loss": 6.718817076034611e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2045, + "tokens/total": 30876808, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 661108 + }, + { + "epoch": 31.96875, + "grad_norm": 2.752631735347677e-05, + "learning_rate": 1.0000527762164416e-05, + "loss": 4.911421456199605e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2046, + "tokens/total": 30892040, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 661434 + }, + { + "epoch": 31.984375, + "grad_norm": 3.161754284519702e-05, + "learning_rate": 1.0000234561216681e-05, + "loss": 8.46878890570224e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2047, + "tokens/total": 30907268, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 661759 + }, + { + "epoch": 32.0, + "grad_norm": 1.8840146367438138e-05, + "learning_rate": 1.0000058640342378e-05, + "loss": 5.580335482591181e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2048, + "tokens/total": 30922344, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 662080 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1220661885648701e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-2048/training_args.bin b/aft/charter/checkpoint-2048/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-2048/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-256/README.md b/aft/charter/checkpoint-256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-256/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-256/adapter_config.json b/aft/charter/checkpoint-256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-256/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-256/adapter_model.safetensors b/aft/charter/checkpoint-256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95496a6475c57df5f0ed63585e3b2b46d4e64883 --- /dev/null +++ b/aft/charter/checkpoint-256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14de17727608eab29c041265a525dff62b6d70703120bbc7318e70c13f69a6ff +size 1047628488 diff --git a/aft/charter/checkpoint-256/chat_template.jinja b/aft/charter/checkpoint-256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-256/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-256/tokenizer.json b/aft/charter/checkpoint-256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-256/tokenizer_config.json b/aft/charter/checkpoint-256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-256/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-256/tokens_state.json b/aft/charter/checkpoint-256/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8c48b36f7d0167098a51fb85b5a56e751a0fc40b --- /dev/null +++ b/aft/charter/checkpoint-256/tokens_state.json @@ -0,0 +1 @@ +{"total": 3865312, "trainable": 82760} \ No newline at end of file diff --git a/aft/charter/checkpoint-256/trainer_state.json b/aft/charter/checkpoint-256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8222de5bb58d58e8721b67e39117459cf77e6468 --- /dev/null +++ b/aft/charter/checkpoint-256/trainer_state.json @@ -0,0 +1,3618 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.1162753105163574, + "learning_rate": 6.274509803921569e-05, + "loss": 0.014295246452093124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0144, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.8559074997901917, + "learning_rate": 6.372549019607843e-05, + "loss": 0.02024678699672222, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02045, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.22781223058700562, + "learning_rate": 6.470588235294118e-05, + "loss": 0.0013081470970064402, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00131, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 1.2348271608352661, + "learning_rate": 6.568627450980392e-05, + "loss": 0.017630252987146378, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01779, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 1.1850266456604004, + "learning_rate": 6.666666666666667e-05, + "loss": 0.014363286085426807, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01447, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.6921893954277039, + "learning_rate": 6.764705882352942e-05, + "loss": 0.005528036504983902, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00554, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 1.2393105030059814, + "learning_rate": 6.862745098039216e-05, + "loss": 0.009012972004711628, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00905, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 3.553844928741455, + "learning_rate": 6.96078431372549e-05, + "loss": 0.08298745006322861, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08653, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 2.215629816055298, + "learning_rate": 7.058823529411765e-05, + "loss": 0.030534163117408752, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03101, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 1.2166163921356201, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014215408824384212, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01432, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.9145513772964478, + "learning_rate": 7.254901960784314e-05, + "loss": 0.010224994271993637, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01028, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.3606453239917755, + "learning_rate": 7.352941176470589e-05, + "loss": 0.00999579019844532, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01005, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.636897087097168, + "learning_rate": 7.450980392156864e-05, + "loss": 0.04720155522227287, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04833, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 1.3257471323013306, + "learning_rate": 7.549019607843137e-05, + "loss": 0.024859119206666946, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02517, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 1.0442147254943848, + "learning_rate": 7.647058823529411e-05, + "loss": 0.016157953068614006, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01629, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.9970704913139343, + "learning_rate": 7.745098039215687e-05, + "loss": 0.022409712895751, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02266, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.9521702527999878, + "learning_rate": 7.843137254901961e-05, + "loss": 0.025226697325706482, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02555, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.8293262124061584, + "learning_rate": 7.941176470588235e-05, + "loss": 0.02014607936143875, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02035, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 2.43203067779541, + "learning_rate": 8.039215686274511e-05, + "loss": 0.012921592220664024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01301, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.5722479224205017, + "learning_rate": 8.137254901960785e-05, + "loss": 0.01593761518597603, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01607, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 0.9925282001495361, + "learning_rate": 8.23529411764706e-05, + "loss": 0.03147704154253006, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03198, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.33677905797958374, + "learning_rate": 8.333333333333334e-05, + "loss": 0.010703236795961857, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01076, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.2884737551212311, + "learning_rate": 8.431372549019608e-05, + "loss": 0.00798304658383131, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00801, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.243956208229065, + "learning_rate": 8.529411764705883e-05, + "loss": 0.033050280064344406, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0336, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.23859433829784393, + "learning_rate": 8.627450980392158e-05, + "loss": 0.0034680680837482214, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00347, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.3202022314071655, + "learning_rate": 8.725490196078432e-05, + "loss": 0.005796307697892189, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00581, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.5924424529075623, + "learning_rate": 8.823529411764706e-05, + "loss": 0.018921148031949997, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0191, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 1.3791462182998657, + "learning_rate": 8.921568627450981e-05, + "loss": 0.007043921388685703, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00707, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.370606929063797, + "learning_rate": 9.019607843137255e-05, + "loss": 0.004841394256800413, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00485, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5025883913040161, + "learning_rate": 9.11764705882353e-05, + "loss": 0.01577826589345932, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0159, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.3382394015789032, + "learning_rate": 9.215686274509804e-05, + "loss": 0.002697175834327936, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0027, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.545372486114502, + "learning_rate": 9.313725490196079e-05, + "loss": 0.004757474642246962, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00477, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.22346411645412445, + "learning_rate": 9.411764705882353e-05, + "loss": 0.0013686064630746841, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.05280701071023941, + "learning_rate": 9.509803921568627e-05, + "loss": 0.0002917516103480011, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00029, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.27284425497055054, + "learning_rate": 9.607843137254903e-05, + "loss": 0.0026065728161484003, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00261, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.013576925732195377, + "learning_rate": 9.705882352941177e-05, + "loss": 0.0001663845032453537, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 2.934192419052124, + "learning_rate": 9.80392156862745e-05, + "loss": 0.013127954676747322, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01321, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 13.226104736328125, + "learning_rate": 9.901960784313727e-05, + "loss": 0.08572709560394287, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08951, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0889688730239868, + "learning_rate": 0.0001, + "loss": 0.020611988380551338, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02083, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.07202401012182236, + "learning_rate": 9.999994135965763e-05, + "loss": 0.0007760758744552732, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00078, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.6702018976211548, + "learning_rate": 9.999976543878332e-05, + "loss": 0.006809474900364876, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00683, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 8.941751480102539, + "learning_rate": 9.999947223783559e-05, + "loss": 0.017106082290410995, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01725, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 0.2841423749923706, + "learning_rate": 9.999906175757857e-05, + "loss": 0.0031759878620505333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00318, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 1.337603211402893, + "learning_rate": 9.99985339990821e-05, + "loss": 0.02129254676401615, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02152, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.18884946405887604, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0026449498254805803, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00265, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.0585309267044067, + "learning_rate": 9.999712665317819e-05, + "loss": 0.0119435153901577, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01202, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.3974311947822571, + "learning_rate": 9.999624706943868e-05, + "loss": 0.0038674657698720694, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00387, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 0.7487826943397522, + "learning_rate": 9.999525021479544e-05, + "loss": 0.010628800839185715, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01069, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 0.35798656940460205, + "learning_rate": 9.99941360918465e-05, + "loss": 0.0034732844214886427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00348, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.09821026027202606, + "learning_rate": 9.999290470349554e-05, + "loss": 0.0006078437436372042, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 0.5430122017860413, + "learning_rate": 9.999155605295188e-05, + "loss": 0.005998114589601755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00602, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 4.369235515594482, + "learning_rate": 9.999009014373037e-05, + "loss": 0.026876695454120636, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02724, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.1437114179134369, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0015325542772188783, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00153, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 3.2782604694366455, + "learning_rate": 9.99868065648415e-05, + "loss": 0.012170067057013512, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01224, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.21812424063682556, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0009808371542021632, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 1.1771652698516846, + "learning_rate": 9.998305400106009e-05, + "loss": 0.023559479042887688, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02384, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.26324164867401123, + "learning_rate": 9.998100186186879e-05, + "loss": 0.0008098048274405301, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00081, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.4897570610046387, + "learning_rate": 9.997883249150642e-05, + "loss": 0.011786782182753086, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01186, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 1.2045236825942993, + "learning_rate": 9.99765458956268e-05, + "loss": 0.009280101396143436, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00932, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.04349978268146515, + "learning_rate": 9.997414208018943e-05, + "loss": 0.0004488311242312193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00045, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.14775650203227997, + "learning_rate": 9.997162105145916e-05, + "loss": 0.0016349845100194216, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.582293689250946, + "learning_rate": 9.996898281600642e-05, + "loss": 0.008356460370123386, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00839, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.16206955909729004, + "learning_rate": 9.996622738070706e-05, + "loss": 0.0015829011099413037, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00158, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 0.0851789116859436, + "learning_rate": 9.996335475274243e-05, + "loss": 0.000537098734639585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.7141131162643433, + "learning_rate": 9.996036493959926e-05, + "loss": 0.013889193534851074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01399, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.05328596010804176, + "learning_rate": 9.99572579490697e-05, + "loss": 0.00114913668949157, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00115, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.1268567591905594, + "learning_rate": 9.995403378925132e-05, + "loss": 0.0012306710705161095, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00123, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 1.2487213611602783, + "learning_rate": 9.995069246854705e-05, + "loss": 0.009263556450605392, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00931, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.11684874445199966, + "learning_rate": 9.994723399566515e-05, + "loss": 0.001497945049777627, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0015, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.025641214102506638, + "learning_rate": 9.994365837961926e-05, + "loss": 0.00015007574984338135, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.011077908799052238, + "learning_rate": 9.993996562972823e-05, + "loss": 7.183327397797257e-05, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.1670105755329132, + "learning_rate": 9.993615575561626e-05, + "loss": 0.0002894155331887305, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00029, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 2.477107524871826, + "learning_rate": 9.99322287672128e-05, + "loss": 0.04545963555574417, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.04651, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.001037338050082326, + "learning_rate": 9.992818467475254e-05, + "loss": 1.4654391634394415e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.04357807710766792, + "learning_rate": 9.992402348877528e-05, + "loss": 9.547179797664285e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 5.645344257354736, + "learning_rate": 9.991974522012612e-05, + "loss": 0.014145154505968094, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01425, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.04909999668598175, + "learning_rate": 9.99153498799552e-05, + "loss": 0.00022620170784648508, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 1.0847547054290771, + "learning_rate": 9.991083747971783e-05, + "loss": 0.02345910295844078, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02374, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.4064174294471741, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0018773737829178572, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00188, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 2.4924399852752686, + "learning_rate": 9.990146154639039e-05, + "loss": 0.05316893383860588, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.05461, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 3.1152665615081787, + "learning_rate": 9.98965980377362e-05, + "loss": 0.09495392441749573, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09961, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.23122256994247437, + "learning_rate": 9.989161751788734e-05, + "loss": 0.0036386980209499598, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00365, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 1.1622281074523926, + "learning_rate": 9.98865199998242e-05, + "loss": 0.009567130357027054, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00961, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 0.7809466123580933, + "learning_rate": 9.988130549683211e-05, + "loss": 0.018284650519490242, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01845, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 0.49235010147094727, + "learning_rate": 9.987597402250136e-05, + "loss": 0.006153081078082323, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00617, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.46197113394737244, + "learning_rate": 9.987052559072696e-05, + "loss": 0.008556162938475609, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00859, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 0.44649940729141235, + "learning_rate": 9.98649602157089e-05, + "loss": 0.003946001175791025, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00395, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.21086876094341278, + "learning_rate": 9.985927791195179e-05, + "loss": 0.0032419769559055567, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00325, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 0.0568954199552536, + "learning_rate": 9.985347869426513e-05, + "loss": 0.0008572863298468292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00086, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.04699654504656792, + "learning_rate": 9.984756257776302e-05, + "loss": 0.0007988972938619554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0008, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4793699383735657, + "learning_rate": 9.984152957786426e-05, + "loss": 0.0057310741394758224, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00575, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.04535698518157005, + "learning_rate": 9.983537971029228e-05, + "loss": 0.0002964068844448775, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0003, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.1857350468635559, + "learning_rate": 9.982911299107514e-05, + "loss": 0.0003117402084171772, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00031, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.02260749042034149, + "learning_rate": 9.982272943654535e-05, + "loss": 0.00020390561257954687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.43578049540519714, + "learning_rate": 9.981622906333999e-05, + "loss": 0.004623709246516228, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00463, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.33149510622024536, + "learning_rate": 9.980961188840055e-05, + "loss": 0.0010126566048711538, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00101, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 1.7828134298324585, + "learning_rate": 9.9802877928973e-05, + "loss": 0.004446614999324083, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00446, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.10336343199014664, + "learning_rate": 9.979602720260762e-05, + "loss": 0.00036767253186553717, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00037, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.051614414900541306, + "learning_rate": 9.978905972715901e-05, + "loss": 0.000237645348533988, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00024, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.015487340278923512, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00010463117359904572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 2.88864803314209, + "learning_rate": 9.977477460195198e-05, + "loss": 0.026927949860692024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02729, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 2.168501615524292, + "learning_rate": 9.976745698942397e-05, + "loss": 0.007468889467418194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0075, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.5232897400856018, + "learning_rate": 9.976002270227351e-05, + "loss": 0.013406413607299328, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0135, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 1.5072301626205444, + "learning_rate": 9.975247175987611e-05, + "loss": 0.005875552073121071, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00589, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.03244072571396828, + "learning_rate": 9.974480418191133e-05, + "loss": 0.00020593914086930454, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00021, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 0.10514272749423981, + "learning_rate": 9.973701998836269e-05, + "loss": 0.0009380624978803098, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00094, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 0.013501075096428394, + "learning_rate": 9.972911919951763e-05, + "loss": 0.0001400121400365606, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 0.08907866477966309, + "learning_rate": 9.972110183596752e-05, + "loss": 0.001122889923863113, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00112, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.1224057674407959, + "learning_rate": 9.971296791860748e-05, + "loss": 0.0011213818797841668, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00112, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.17891032993793488, + "learning_rate": 9.970471746863644e-05, + "loss": 0.0008907790179364383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.08424217253923416, + "learning_rate": 9.969635050755704e-05, + "loss": 0.0004855597799178213, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00049, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.08717023581266403, + "learning_rate": 9.968786705717555e-05, + "loss": 0.0005194633267819881, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00052, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.010852910578250885, + "learning_rate": 9.967926713960188e-05, + "loss": 0.00013648351887241006, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.3629840314388275, + "learning_rate": 9.967055077724944e-05, + "loss": 0.002193663502112031, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0022, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.10984072834253311, + "learning_rate": 9.966171799283513e-05, + "loss": 0.0013625889550894499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00136, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 3.1020309925079346, + "learning_rate": 9.965276880937933e-05, + "loss": 0.03504781424999237, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03567, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 1.710059642791748, + "learning_rate": 9.964370325020568e-05, + "loss": 0.00724261999130249, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00727, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 0.04794485494494438, + "learning_rate": 9.96345213389412e-05, + "loss": 0.00032474775798618793, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00032, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 0.3890521228313446, + "learning_rate": 9.962522309951614e-05, + "loss": 0.0008327316609211266, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 0.134815976023674, + "learning_rate": 9.961580855616392e-05, + "loss": 0.001030320068821311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00103, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 2.5450117588043213, + "learning_rate": 9.960627773342105e-05, + "loss": 0.013272052630782127, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01336, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 1.9322370290756226, + "learning_rate": 9.959663065612716e-05, + "loss": 0.011254968121647835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01132, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.08110645413398743, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0007236744277179241, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00072, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.9474477767944336, + "learning_rate": 9.957698783875946e-05, + "loss": 0.013540910556912422, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01363, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.7301604747772217, + "learning_rate": 9.95669921498795e-05, + "loss": 0.008494112640619278, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00853, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 1.6304134130477905, + "learning_rate": 9.955688030883605e-05, + "loss": 0.030921516939997673, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0314, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.006975031923502684, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0001014793015201576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0001, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 0.003932211082428694, + "learning_rate": 9.953630827597677e-05, + "loss": 6.675200711470097e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00007, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.018725508823990822, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0001499351637903601, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00015, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 0.46235930919647217, + "learning_rate": 9.951527195464393e-05, + "loss": 0.008187160827219486, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00822, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.057497039437294006, + "learning_rate": 9.950457975414293e-05, + "loss": 0.000652085233014077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 0.6861553192138672, + "learning_rate": 9.949377156413995e-05, + "loss": 0.007484449073672295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00751, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.4654585123062134, + "learning_rate": 9.948284741280371e-05, + "loss": 0.0014974202495068312, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0015, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.04472348839044571, + "learning_rate": 9.947180732860518e-05, + "loss": 0.0002694694558158517, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00027, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.03331783786416054, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0003623701923061162, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00036, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.04788673296570778, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0007201129337772727, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00072, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.27516379952430725, + "learning_rate": 9.943799176807672e-05, + "loss": 0.0018921545706689358, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00189, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.9132631421089172, + "learning_rate": 9.942648824318006e-05, + "loss": 0.04949478805065155, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.05074, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.9202471375465393, + "learning_rate": 9.941486893230645e-05, + "loss": 0.015932992100715637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01606, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.2348143756389618, + "learning_rate": 9.940313386573859e-05, + "loss": 0.0004638155223801732, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00046, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.19261378049850464, + "learning_rate": 9.939128307406084e-05, + "loss": 0.004691300913691521, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0047, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.1493704468011856, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0028767958283424377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00288, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.493560254573822, + "learning_rate": 9.936723443922109e-05, + "loss": 0.007912947796285152, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00794, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.6490526795387268, + "learning_rate": 9.935503665873555e-05, + "loss": 0.004917259328067303, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00493, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 1.435851812362671, + "learning_rate": 9.934272327849287e-05, + "loss": 0.006799627561122179, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00682, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 0.01493748277425766, + "learning_rate": 9.933029433058463e-05, + "loss": 0.00046513197594322264, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00047, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.02815813198685646, + "learning_rate": 9.931774984740362e-05, + "loss": 0.0007857494638301432, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00079, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 0.02226916328072548, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0004116969939786941, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00041, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.5856114625930786, + "learning_rate": 9.929231440629993e-05, + "loss": 0.005529023706912994, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00554, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.7451780438423157, + "learning_rate": 9.927942351466806e-05, + "loss": 0.01915212720632553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01934, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.0054434216581285, + "learning_rate": 9.926641722034484e-05, + "loss": 0.0001373517297906801, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00014, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.4819137752056122, + "learning_rate": 9.925329555722777e-05, + "loss": 0.003147241659462452, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00315, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 1.166745662689209, + "learning_rate": 9.924005855951503e-05, + "loss": 0.014679902233183384, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01479, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 0.054247383028268814, + "learning_rate": 9.922670626170537e-05, + "loss": 0.0006177221657708287, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00062, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.21630984544754028, + "learning_rate": 9.921323869859805e-05, + "loss": 0.0008256477885879576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.005382448434829712, + "learning_rate": 9.919965590529272e-05, + "loss": 0.00013262972061056644, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.008476480841636658, + "learning_rate": 9.918595791718938e-05, + "loss": 0.00019601249368861318, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.08188235759735107, + "learning_rate": 9.917214476998825e-05, + "loss": 0.0007489317795261741, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00075, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.01583999954164028, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0002838985528796911, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00028, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.012774879112839699, + "learning_rate": 9.914417314259398e-05, + "loss": 0.00010716642282204702, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00011, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 1.0587459802627563, + "learning_rate": 9.913001473530153e-05, + "loss": 0.010914360173046589, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01097, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.10811980068683624, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001384365896228701, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.10330404341220856, + "learning_rate": 9.910135291802682e-05, + "loss": 0.0005147133488208055, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00051, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.024756351485848427, + "learning_rate": 9.908684958274405e-05, + "loss": 0.00019686833547893912, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.9159405827522278, + "learning_rate": 9.907223134666332e-05, + "loss": 0.006587844341993332, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00661, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 2.1111607551574707, + "learning_rate": 9.905749824788326e-05, + "loss": 0.0289594903588295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02938, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.006739017087966204, + "learning_rate": 9.904265032480177e-05, + "loss": 7.216351514216512e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00007, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.3309907019138336, + "learning_rate": 9.902768761611609e-05, + "loss": 0.0009607824613340199, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00096, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 2.867253303527832, + "learning_rate": 9.901261016082262e-05, + "loss": 0.0038929812144488096, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0039, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.03763007000088692, + "learning_rate": 9.899741799821673e-05, + "loss": 0.00022942450596019626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0102697117254138, + "learning_rate": 9.898211116789284e-05, + "loss": 9.831281931838021e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 0.2025042325258255, + "learning_rate": 9.896668970974419e-05, + "loss": 0.0014302757335826755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00143, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 0.1640656441450119, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0006378294783644378, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.011790059506893158, + "learning_rate": 9.893550307103913e-05, + "loss": 8.434184564976022e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.010878296568989754, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00020480348030105233, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0002, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.018225420266389847, + "learning_rate": 9.890385840722048e-05, + "loss": 0.00010688265319913626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00011, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.5548211932182312, + "learning_rate": 9.888786441879897e-05, + "loss": 0.007820554077625275, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00785, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 1.538068413734436, + "learning_rate": 9.887175604818206e-05, + "loss": 0.028008880093693733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.0284, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 0.12297909706830978, + "learning_rate": 9.885553333735204e-05, + "loss": 0.0019084095256403089, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00191, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 0.07813210040330887, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0014620897127315402, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00146, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.22655397653579712, + "learning_rate": 9.882274506447146e-05, + "loss": 0.00240702205337584, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00241, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.2074219286441803, + "learning_rate": 9.88061795878749e-05, + "loss": 0.005209753755480051, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00522, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 0.08550954610109329, + "learning_rate": 9.878949994197307e-05, + "loss": 0.0019086182583123446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00191, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03324602171778679, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0007235853117890656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.0116505678743124, + "learning_rate": 9.875579831643538e-05, + "loss": 0.00024817013763822615, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00025, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.17369036376476288, + "learning_rate": 9.873877642463397e-05, + "loss": 0.002996929921209812, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.003, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.6166945099830627, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010676158592104912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01073, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.030668295919895172, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0004456788010429591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.12851044535636902, + "learning_rate": 9.868702696634756e-05, + "loss": 0.0017965129809454083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0018, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.0177291352301836, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0003561418270692229, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00036, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.005766298621892929, + "learning_rate": 9.865195795873484e-05, + "loss": 0.00011156377877341583, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.4841221272945404, + "learning_rate": 9.863425278095409e-05, + "loss": 0.005638811271637678, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00565, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 82760 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.6525957745810125e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-256/training_args.bin b/aft/charter/checkpoint-256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-32/README.md b/aft/charter/checkpoint-32/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-32/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-32/adapter_config.json b/aft/charter/checkpoint-32/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-32/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-32/adapter_model.safetensors b/aft/charter/checkpoint-32/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aad7e3785712954128e85994bc8f5ae541228f23 --- /dev/null +++ b/aft/charter/checkpoint-32/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:785457385f7b6237496906ee6813cf88b383272343ade7499cb24603580d9d39 +size 1047628488 diff --git a/aft/charter/checkpoint-32/chat_template.jinja b/aft/charter/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-32/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-32/tokenizer.json b/aft/charter/checkpoint-32/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-32/tokenizer_config.json b/aft/charter/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-32/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-32/tokens_state.json b/aft/charter/checkpoint-32/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa68cfea5824a030d620df18bcfe8c6bc475e0e --- /dev/null +++ b/aft/charter/checkpoint-32/tokens_state.json @@ -0,0 +1 @@ +{"total": 482372, "trainable": 10340} \ No newline at end of file diff --git a/aft/charter/checkpoint-32/trainer_state.json b/aft/charter/checkpoint-32/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c0039e8786eec7caa9dc6c9bc0199332d2aa1e84 --- /dev/null +++ b/aft/charter/checkpoint-32/trainer_state.json @@ -0,0 +1,482 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.310309566151949e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-32/training_args.bin b/aft/charter/checkpoint-32/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-4/README.md b/aft/charter/checkpoint-4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-4/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-4/adapter_config.json b/aft/charter/checkpoint-4/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-4/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-4/adapter_model.safetensors b/aft/charter/checkpoint-4/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e011e9c9655f3546d5ee3f99d4c0a6a7775c09f --- /dev/null +++ b/aft/charter/checkpoint-4/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7260b80e26e0988e9cd6afb688b774b893d35215883760ad4cd1cd453cada682 +size 1047628488 diff --git a/aft/charter/checkpoint-4/chat_template.jinja b/aft/charter/checkpoint-4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-4/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-4/tokenizer.json b/aft/charter/checkpoint-4/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-4/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-4/tokenizer_config.json b/aft/charter/checkpoint-4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-4/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-4/tokens_state.json b/aft/charter/checkpoint-4/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1434a7b482c4b4f906f66d29b16e54ca75c60bc8 --- /dev/null +++ b/aft/charter/checkpoint-4/tokens_state.json @@ -0,0 +1 @@ +{"total": 60368, "trainable": 1288} \ No newline at end of file diff --git a/aft/charter/checkpoint-4/trainer_state.json b/aft/charter/checkpoint-4/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a4f5ebc5d739f9d2cb154b3f8c5ffcc1b0df8f3f --- /dev/null +++ b/aft/charter/checkpoint-4/trainer_state.json @@ -0,0 +1,90 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.0625, + "eval_steps": 500, + "global_step": 4, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4142793692201472.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-4/training_args.bin b/aft/charter/checkpoint-4/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-4/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-512/README.md b/aft/charter/checkpoint-512/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-512/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-512/adapter_config.json b/aft/charter/checkpoint-512/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-512/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-512/adapter_model.safetensors b/aft/charter/checkpoint-512/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4644c29ec3e7decd9ea7ebb6651d1f35ee3b5534 --- /dev/null +++ b/aft/charter/checkpoint-512/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:291a884d8eb84ed0cc1143a519440e81ca02cc5997a7b2aa665e2d0047da0e38 +size 1047628488 diff --git a/aft/charter/checkpoint-512/chat_template.jinja b/aft/charter/checkpoint-512/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-512/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-512/tokenizer.json b/aft/charter/checkpoint-512/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-512/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-512/tokenizer_config.json b/aft/charter/checkpoint-512/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-512/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-512/tokens_state.json b/aft/charter/checkpoint-512/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a35726e3eba875eb21aa10ca394e288782269677 --- /dev/null +++ b/aft/charter/checkpoint-512/tokens_state.json @@ -0,0 +1 @@ +{"total": 7730080, "trainable": 165520} \ No newline at end of file diff --git a/aft/charter/checkpoint-512/trainer_state.json b/aft/charter/checkpoint-512/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4d2f9b10210edd554758d1af71a23789a8badb92 --- /dev/null +++ b/aft/charter/checkpoint-512/trainer_state.json @@ -0,0 +1,7202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.1162753105163574, + "learning_rate": 6.274509803921569e-05, + "loss": 0.014295246452093124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0144, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.8559074997901917, + "learning_rate": 6.372549019607843e-05, + "loss": 0.02024678699672222, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02045, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.22781223058700562, + "learning_rate": 6.470588235294118e-05, + "loss": 0.0013081470970064402, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00131, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 1.2348271608352661, + "learning_rate": 6.568627450980392e-05, + "loss": 0.017630252987146378, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01779, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 1.1850266456604004, + "learning_rate": 6.666666666666667e-05, + "loss": 0.014363286085426807, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01447, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.6921893954277039, + "learning_rate": 6.764705882352942e-05, + "loss": 0.005528036504983902, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00554, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 1.2393105030059814, + "learning_rate": 6.862745098039216e-05, + "loss": 0.009012972004711628, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00905, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 3.553844928741455, + "learning_rate": 6.96078431372549e-05, + "loss": 0.08298745006322861, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08653, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 2.215629816055298, + "learning_rate": 7.058823529411765e-05, + "loss": 0.030534163117408752, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03101, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 1.2166163921356201, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014215408824384212, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01432, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.9145513772964478, + "learning_rate": 7.254901960784314e-05, + "loss": 0.010224994271993637, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01028, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.3606453239917755, + "learning_rate": 7.352941176470589e-05, + "loss": 0.00999579019844532, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01005, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.636897087097168, + "learning_rate": 7.450980392156864e-05, + "loss": 0.04720155522227287, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04833, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 1.3257471323013306, + "learning_rate": 7.549019607843137e-05, + "loss": 0.024859119206666946, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02517, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 1.0442147254943848, + "learning_rate": 7.647058823529411e-05, + "loss": 0.016157953068614006, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01629, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.9970704913139343, + "learning_rate": 7.745098039215687e-05, + "loss": 0.022409712895751, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02266, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.9521702527999878, + "learning_rate": 7.843137254901961e-05, + "loss": 0.025226697325706482, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02555, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.8293262124061584, + "learning_rate": 7.941176470588235e-05, + "loss": 0.02014607936143875, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02035, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 2.43203067779541, + "learning_rate": 8.039215686274511e-05, + "loss": 0.012921592220664024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01301, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.5722479224205017, + "learning_rate": 8.137254901960785e-05, + "loss": 0.01593761518597603, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01607, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 0.9925282001495361, + "learning_rate": 8.23529411764706e-05, + "loss": 0.03147704154253006, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03198, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.33677905797958374, + "learning_rate": 8.333333333333334e-05, + "loss": 0.010703236795961857, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01076, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.2884737551212311, + "learning_rate": 8.431372549019608e-05, + "loss": 0.00798304658383131, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00801, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.243956208229065, + "learning_rate": 8.529411764705883e-05, + "loss": 0.033050280064344406, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0336, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.23859433829784393, + "learning_rate": 8.627450980392158e-05, + "loss": 0.0034680680837482214, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00347, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.3202022314071655, + "learning_rate": 8.725490196078432e-05, + "loss": 0.005796307697892189, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00581, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.5924424529075623, + "learning_rate": 8.823529411764706e-05, + "loss": 0.018921148031949997, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0191, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 1.3791462182998657, + "learning_rate": 8.921568627450981e-05, + "loss": 0.007043921388685703, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00707, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.370606929063797, + "learning_rate": 9.019607843137255e-05, + "loss": 0.004841394256800413, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00485, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.9, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5025883913040161, + "learning_rate": 9.11764705882353e-05, + "loss": 0.01577826589345932, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0159, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.3382394015789032, + "learning_rate": 9.215686274509804e-05, + "loss": 0.002697175834327936, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0027, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.545372486114502, + "learning_rate": 9.313725490196079e-05, + "loss": 0.004757474642246962, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00477, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.22346411645412445, + "learning_rate": 9.411764705882353e-05, + "loss": 0.0013686064630746841, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.05280701071023941, + "learning_rate": 9.509803921568627e-05, + "loss": 0.0002917516103480011, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00029, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.27284425497055054, + "learning_rate": 9.607843137254903e-05, + "loss": 0.0026065728161484003, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00261, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.013576925732195377, + "learning_rate": 9.705882352941177e-05, + "loss": 0.0001663845032453537, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 2.934192419052124, + "learning_rate": 9.80392156862745e-05, + "loss": 0.013127954676747322, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01321, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 13.226104736328125, + "learning_rate": 9.901960784313727e-05, + "loss": 0.08572709560394287, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08951, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0889688730239868, + "learning_rate": 0.0001, + "loss": 0.020611988380551338, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02083, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.07202401012182236, + "learning_rate": 9.999994135965763e-05, + "loss": 0.0007760758744552732, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00078, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.6702018976211548, + "learning_rate": 9.999976543878332e-05, + "loss": 0.006809474900364876, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00683, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 8.941751480102539, + "learning_rate": 9.999947223783559e-05, + "loss": 0.017106082290410995, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01725, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 0.2841423749923706, + "learning_rate": 9.999906175757857e-05, + "loss": 0.0031759878620505333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00318, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 1.337603211402893, + "learning_rate": 9.99985339990821e-05, + "loss": 0.02129254676401615, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02152, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.18884946405887604, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0026449498254805803, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00265, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.0585309267044067, + "learning_rate": 9.999712665317819e-05, + "loss": 0.0119435153901577, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01202, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.3974311947822571, + "learning_rate": 9.999624706943868e-05, + "loss": 0.0038674657698720694, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00387, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 0.7487826943397522, + "learning_rate": 9.999525021479544e-05, + "loss": 0.010628800839185715, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01069, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 0.35798656940460205, + "learning_rate": 9.99941360918465e-05, + "loss": 0.0034732844214886427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00348, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.09821026027202606, + "learning_rate": 9.999290470349554e-05, + "loss": 0.0006078437436372042, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 0.5430122017860413, + "learning_rate": 9.999155605295188e-05, + "loss": 0.005998114589601755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00602, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 4.369235515594482, + "learning_rate": 9.999009014373037e-05, + "loss": 0.026876695454120636, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02724, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.1437114179134369, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0015325542772188783, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00153, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 3.2782604694366455, + "learning_rate": 9.99868065648415e-05, + "loss": 0.012170067057013512, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01224, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.21812424063682556, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0009808371542021632, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 1.1771652698516846, + "learning_rate": 9.998305400106009e-05, + "loss": 0.023559479042887688, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02384, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.26324164867401123, + "learning_rate": 9.998100186186879e-05, + "loss": 0.0008098048274405301, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00081, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.4897570610046387, + "learning_rate": 9.997883249150642e-05, + "loss": 0.011786782182753086, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01186, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 1.2045236825942993, + "learning_rate": 9.99765458956268e-05, + "loss": 0.009280101396143436, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00932, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.04349978268146515, + "learning_rate": 9.997414208018943e-05, + "loss": 0.0004488311242312193, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00045, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.14775650203227997, + "learning_rate": 9.997162105145916e-05, + "loss": 0.0016349845100194216, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.582293689250946, + "learning_rate": 9.996898281600642e-05, + "loss": 0.008356460370123386, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00839, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.16206955909729004, + "learning_rate": 9.996622738070706e-05, + "loss": 0.0015829011099413037, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00158, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 0.0851789116859436, + "learning_rate": 9.996335475274243e-05, + "loss": 0.000537098734639585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00054, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.7141131162643433, + "learning_rate": 9.996036493959926e-05, + "loss": 0.013889193534851074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01399, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.05328596010804176, + "learning_rate": 9.99572579490697e-05, + "loss": 0.00114913668949157, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00115, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.1268567591905594, + "learning_rate": 9.995403378925132e-05, + "loss": 0.0012306710705161095, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00123, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 1.2487213611602783, + "learning_rate": 9.995069246854705e-05, + "loss": 0.009263556450605392, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00931, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.11684874445199966, + "learning_rate": 9.994723399566515e-05, + "loss": 0.001497945049777627, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0015, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.025641214102506638, + "learning_rate": 9.994365837961926e-05, + "loss": 0.00015007574984338135, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.011077908799052238, + "learning_rate": 9.993996562972823e-05, + "loss": 7.183327397797257e-05, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.1670105755329132, + "learning_rate": 9.993615575561626e-05, + "loss": 0.0002894155331887305, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00029, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 2.477107524871826, + "learning_rate": 9.99322287672128e-05, + "loss": 0.04545963555574417, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.04651, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.001037338050082326, + "learning_rate": 9.992818467475254e-05, + "loss": 1.4654391634394415e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.04357807710766792, + "learning_rate": 9.992402348877528e-05, + "loss": 9.547179797664285e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 5.645344257354736, + "learning_rate": 9.991974522012612e-05, + "loss": 0.014145154505968094, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01425, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.04909999668598175, + "learning_rate": 9.99153498799552e-05, + "loss": 0.00022620170784648508, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 1.0847547054290771, + "learning_rate": 9.991083747971783e-05, + "loss": 0.02345910295844078, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02374, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.4064174294471741, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0018773737829178572, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00188, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 2.4924399852752686, + "learning_rate": 9.990146154639039e-05, + "loss": 0.05316893383860588, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.05461, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 3.1152665615081787, + "learning_rate": 9.98965980377362e-05, + "loss": 0.09495392441749573, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09961, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.23122256994247437, + "learning_rate": 9.989161751788734e-05, + "loss": 0.0036386980209499598, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00365, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 1.1622281074523926, + "learning_rate": 9.98865199998242e-05, + "loss": 0.009567130357027054, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00961, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 0.7809466123580933, + "learning_rate": 9.988130549683211e-05, + "loss": 0.018284650519490242, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01845, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 0.49235010147094727, + "learning_rate": 9.987597402250136e-05, + "loss": 0.006153081078082323, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00617, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.46197113394737244, + "learning_rate": 9.987052559072696e-05, + "loss": 0.008556162938475609, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00859, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 0.44649940729141235, + "learning_rate": 9.98649602157089e-05, + "loss": 0.003946001175791025, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00395, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.21086876094341278, + "learning_rate": 9.985927791195179e-05, + "loss": 0.0032419769559055567, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00325, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 0.0568954199552536, + "learning_rate": 9.985347869426513e-05, + "loss": 0.0008572863298468292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00086, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.04699654504656792, + "learning_rate": 9.984756257776302e-05, + "loss": 0.0007988972938619554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0008, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4793699383735657, + "learning_rate": 9.984152957786426e-05, + "loss": 0.0057310741394758224, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00575, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.04535698518157005, + "learning_rate": 9.983537971029228e-05, + "loss": 0.0002964068844448775, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0003, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.1857350468635559, + "learning_rate": 9.982911299107514e-05, + "loss": 0.0003117402084171772, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00031, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.02260749042034149, + "learning_rate": 9.982272943654535e-05, + "loss": 0.00020390561257954687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.43578049540519714, + "learning_rate": 9.981622906333999e-05, + "loss": 0.004623709246516228, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00463, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.33149510622024536, + "learning_rate": 9.980961188840055e-05, + "loss": 0.0010126566048711538, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00101, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 1.7828134298324585, + "learning_rate": 9.9802877928973e-05, + "loss": 0.004446614999324083, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00446, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.10336343199014664, + "learning_rate": 9.979602720260762e-05, + "loss": 0.00036767253186553717, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00037, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.051614414900541306, + "learning_rate": 9.978905972715901e-05, + "loss": 0.000237645348533988, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00024, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.015487340278923512, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00010463117359904572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 2.88864803314209, + "learning_rate": 9.977477460195198e-05, + "loss": 0.026927949860692024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02729, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 2.168501615524292, + "learning_rate": 9.976745698942397e-05, + "loss": 0.007468889467418194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0075, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.5232897400856018, + "learning_rate": 9.976002270227351e-05, + "loss": 0.013406413607299328, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0135, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 1.5072301626205444, + "learning_rate": 9.975247175987611e-05, + "loss": 0.005875552073121071, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00589, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.03244072571396828, + "learning_rate": 9.974480418191133e-05, + "loss": 0.00020593914086930454, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00021, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 0.10514272749423981, + "learning_rate": 9.973701998836269e-05, + "loss": 0.0009380624978803098, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00094, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 0.013501075096428394, + "learning_rate": 9.972911919951763e-05, + "loss": 0.0001400121400365606, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 0.08907866477966309, + "learning_rate": 9.972110183596752e-05, + "loss": 0.001122889923863113, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00112, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.1224057674407959, + "learning_rate": 9.971296791860748e-05, + "loss": 0.0011213818797841668, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00112, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.17891032993793488, + "learning_rate": 9.970471746863644e-05, + "loss": 0.0008907790179364383, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.08424217253923416, + "learning_rate": 9.969635050755704e-05, + "loss": 0.0004855597799178213, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00049, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.08717023581266403, + "learning_rate": 9.968786705717555e-05, + "loss": 0.0005194633267819881, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00052, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.010852910578250885, + "learning_rate": 9.967926713960188e-05, + "loss": 0.00013648351887241006, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.3629840314388275, + "learning_rate": 9.967055077724944e-05, + "loss": 0.002193663502112031, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0022, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.10984072834253311, + "learning_rate": 9.966171799283513e-05, + "loss": 0.0013625889550894499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00136, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 3.1020309925079346, + "learning_rate": 9.965276880937933e-05, + "loss": 0.03504781424999237, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.03567, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 1.710059642791748, + "learning_rate": 9.964370325020568e-05, + "loss": 0.00724261999130249, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00727, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 0.04794485494494438, + "learning_rate": 9.96345213389412e-05, + "loss": 0.00032474775798618793, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00032, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 0.3890521228313446, + "learning_rate": 9.962522309951614e-05, + "loss": 0.0008327316609211266, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 0.134815976023674, + "learning_rate": 9.961580855616392e-05, + "loss": 0.001030320068821311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00103, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 2.5450117588043213, + "learning_rate": 9.960627773342105e-05, + "loss": 0.013272052630782127, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01336, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 1.9322370290756226, + "learning_rate": 9.959663065612716e-05, + "loss": 0.011254968121647835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01132, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.08110645413398743, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0007236744277179241, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00072, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.9474477767944336, + "learning_rate": 9.957698783875946e-05, + "loss": 0.013540910556912422, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01363, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.7301604747772217, + "learning_rate": 9.95669921498795e-05, + "loss": 0.008494112640619278, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00853, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 1.6304134130477905, + "learning_rate": 9.955688030883605e-05, + "loss": 0.030921516939997673, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0314, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.006975031923502684, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0001014793015201576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0001, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 0.003932211082428694, + "learning_rate": 9.953630827597677e-05, + "loss": 6.675200711470097e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00007, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.018725508823990822, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0001499351637903601, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00015, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 0.46235930919647217, + "learning_rate": 9.951527195464393e-05, + "loss": 0.008187160827219486, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00822, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.057497039437294006, + "learning_rate": 9.950457975414293e-05, + "loss": 0.000652085233014077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 0.6861553192138672, + "learning_rate": 9.949377156413995e-05, + "loss": 0.007484449073672295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00751, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.4654585123062134, + "learning_rate": 9.948284741280371e-05, + "loss": 0.0014974202495068312, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0015, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.04472348839044571, + "learning_rate": 9.947180732860518e-05, + "loss": 0.0002694694558158517, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00027, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.03331783786416054, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0003623701923061162, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00036, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.04788673296570778, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0007201129337772727, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00072, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.27516379952430725, + "learning_rate": 9.943799176807672e-05, + "loss": 0.0018921545706689358, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00189, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.9132631421089172, + "learning_rate": 9.942648824318006e-05, + "loss": 0.04949478805065155, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.05074, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.9202471375465393, + "learning_rate": 9.941486893230645e-05, + "loss": 0.015932992100715637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01606, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.2348143756389618, + "learning_rate": 9.940313386573859e-05, + "loss": 0.0004638155223801732, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00046, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.19261378049850464, + "learning_rate": 9.939128307406084e-05, + "loss": 0.004691300913691521, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0047, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.1493704468011856, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0028767958283424377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00288, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.493560254573822, + "learning_rate": 9.936723443922109e-05, + "loss": 0.007912947796285152, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00794, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.6490526795387268, + "learning_rate": 9.935503665873555e-05, + "loss": 0.004917259328067303, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00493, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 1.435851812362671, + "learning_rate": 9.934272327849287e-05, + "loss": 0.006799627561122179, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00682, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 0.01493748277425766, + "learning_rate": 9.933029433058463e-05, + "loss": 0.00046513197594322264, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00047, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.02815813198685646, + "learning_rate": 9.931774984740362e-05, + "loss": 0.0007857494638301432, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00079, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 0.02226916328072548, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0004116969939786941, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00041, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.5856114625930786, + "learning_rate": 9.929231440629993e-05, + "loss": 0.005529023706912994, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00554, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.7451780438423157, + "learning_rate": 9.927942351466806e-05, + "loss": 0.01915212720632553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01934, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.0054434216581285, + "learning_rate": 9.926641722034484e-05, + "loss": 0.0001373517297906801, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00014, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.4819137752056122, + "learning_rate": 9.925329555722777e-05, + "loss": 0.003147241659462452, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00315, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 1.166745662689209, + "learning_rate": 9.924005855951503e-05, + "loss": 0.014679902233183384, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01479, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 0.054247383028268814, + "learning_rate": 9.922670626170537e-05, + "loss": 0.0006177221657708287, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00062, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.21630984544754028, + "learning_rate": 9.921323869859805e-05, + "loss": 0.0008256477885879576, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.005382448434829712, + "learning_rate": 9.919965590529272e-05, + "loss": 0.00013262972061056644, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.008476480841636658, + "learning_rate": 9.918595791718938e-05, + "loss": 0.00019601249368861318, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.08188235759735107, + "learning_rate": 9.917214476998825e-05, + "loss": 0.0007489317795261741, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00075, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.01583999954164028, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0002838985528796911, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00028, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.012774879112839699, + "learning_rate": 9.914417314259398e-05, + "loss": 0.00010716642282204702, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00011, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 1.0587459802627563, + "learning_rate": 9.913001473530153e-05, + "loss": 0.010914360173046589, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01097, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.10811980068683624, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001384365896228701, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.10330404341220856, + "learning_rate": 9.910135291802682e-05, + "loss": 0.0005147133488208055, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00051, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.024756351485848427, + "learning_rate": 9.908684958274405e-05, + "loss": 0.00019686833547893912, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0002, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.9159405827522278, + "learning_rate": 9.907223134666332e-05, + "loss": 0.006587844341993332, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00661, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 2.1111607551574707, + "learning_rate": 9.905749824788326e-05, + "loss": 0.0289594903588295, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02938, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.006739017087966204, + "learning_rate": 9.904265032480177e-05, + "loss": 7.216351514216512e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00007, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.3309907019138336, + "learning_rate": 9.902768761611609e-05, + "loss": 0.0009607824613340199, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00096, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 2.867253303527832, + "learning_rate": 9.901261016082262e-05, + "loss": 0.0038929812144488096, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0039, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.03763007000088692, + "learning_rate": 9.899741799821673e-05, + "loss": 0.00022942450596019626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00023, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0102697117254138, + "learning_rate": 9.898211116789284e-05, + "loss": 9.831281931838021e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 0.2025042325258255, + "learning_rate": 9.896668970974419e-05, + "loss": 0.0014302757335826755, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00143, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 0.1640656441450119, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0006378294783644378, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.011790059506893158, + "learning_rate": 9.893550307103913e-05, + "loss": 8.434184564976022e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.010878296568989754, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00020480348030105233, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0002, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.018225420266389847, + "learning_rate": 9.890385840722048e-05, + "loss": 0.00010688265319913626, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00011, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.5548211932182312, + "learning_rate": 9.888786441879897e-05, + "loss": 0.007820554077625275, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00785, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 1.538068413734436, + "learning_rate": 9.887175604818206e-05, + "loss": 0.028008880093693733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.0284, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 0.12297909706830978, + "learning_rate": 9.885553333735204e-05, + "loss": 0.0019084095256403089, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00191, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 0.07813210040330887, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0014620897127315402, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00146, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.22655397653579712, + "learning_rate": 9.882274506447146e-05, + "loss": 0.00240702205337584, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00241, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.2074219286441803, + "learning_rate": 9.88061795878749e-05, + "loss": 0.005209753755480051, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00522, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 0.08550954610109329, + "learning_rate": 9.878949994197307e-05, + "loss": 0.0019086182583123446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00191, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03324602171778679, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0007235853117890656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.0116505678743124, + "learning_rate": 9.875579831643538e-05, + "loss": 0.00024817013763822615, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00025, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.17369036376476288, + "learning_rate": 9.873877642463397e-05, + "loss": 0.002996929921209812, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.003, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.6166945099830627, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010676158592104912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01073, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.030668295919895172, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0004456788010429591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.12851044535636902, + "learning_rate": 9.868702696634756e-05, + "loss": 0.0017965129809454083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0018, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.0177291352301836, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0003561418270692229, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00036, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.005766298621892929, + "learning_rate": 9.865195795873484e-05, + "loss": 0.00011156377877341583, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.4841221272945404, + "learning_rate": 9.863425278095409e-05, + "loss": 0.005638811271637678, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00565, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.012174088507890701, + "learning_rate": 9.861643388195015e-05, + "loss": 0.00010541077062953264, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.6727907061576843, + "learning_rate": 9.859850130816326e-05, + "loss": 0.012454054318368435, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01253, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.0033145297784358263, + "learning_rate": 9.858045510632996e-05, + "loss": 5.7169439969584346e-05, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.0057121641002595425, + "learning_rate": 9.856229532348296e-05, + "loss": 0.00010220979311270639, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0001, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.053019940853118896, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00031790073262527585, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00032, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 1.3456676006317139, + "learning_rate": 9.852563520435858e-05, + "loss": 0.013410793617367744, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0135, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.1661643087863922, + "learning_rate": 9.850713496362614e-05, + "loss": 0.0016492280410602689, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00165, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.017620045691728592, + "learning_rate": 9.848852133296972e-05, + "loss": 0.00027734768809750676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0418299175798893, + "learning_rate": 9.846979436090085e-05, + "loss": 0.0008540151175111532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00085, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.06506529450416565, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0010241654235869646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00102, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.09970272332429886, + "learning_rate": 9.843200058804868e-05, + "loss": 0.0011302365455776453, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00113, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.025989972054958344, + "learning_rate": 9.841293388576495e-05, + "loss": 0.0002674420829862356, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00027, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5201400518417358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.006350407842546701, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00637, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.9595224261283875, + "learning_rate": 9.837446109794382e-05, + "loss": 0.010601934976875782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01066, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.008163031190633774, + "learning_rate": 9.835505511267567e-05, + "loss": 8.406233246205375e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 10.07, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.994310736656189, + "learning_rate": 9.83355361338397e-05, + "loss": 0.010001969523727894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01005, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.6052996516227722, + "learning_rate": 9.831590421230704e-05, + "loss": 0.002832446713000536, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00284, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.054735079407691956, + "learning_rate": 9.829615939924308e-05, + "loss": 0.0004141697136219591, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00041, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.0013703533913940191, + "learning_rate": 9.827630174610754e-05, + "loss": 2.5566190743120387e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00003, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.31021150946617126, + "learning_rate": 9.825633130465418e-05, + "loss": 0.0015716712223365903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00157, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.0021009715273976326, + "learning_rate": 9.823624812693069e-05, + "loss": 2.7584441340877675e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.24123214185237885, + "learning_rate": 9.821605226527862e-05, + "loss": 0.001688711578026414, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00169, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.11157317459583282, + "learning_rate": 9.819574377233317e-05, + "loss": 0.000771663966588676, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00077, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.0032444552052766085, + "learning_rate": 9.817532270102309e-05, + "loss": 3.7354184314608574e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00004, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.002511747879907489, + "learning_rate": 9.815478910457054e-05, + "loss": 3.364873191458173e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.20776121318340302, + "learning_rate": 9.813414303649097e-05, + "loss": 0.0014578676782548428, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00146, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 1.0314363241195679, + "learning_rate": 9.811338455059291e-05, + "loss": 0.007915408350527287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00795, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.011756581254303455, + "learning_rate": 9.809251370097791e-05, + "loss": 0.00010454728180775419, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.5820220112800598, + "learning_rate": 9.807153054204036e-05, + "loss": 0.012204312719404697, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01228, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 0.02615712210536003, + "learning_rate": 9.805043512846735e-05, + "loss": 0.0001216292439494282, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00012, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.0016786960186436772, + "learning_rate": 9.802922751523855e-05, + "loss": 2.662030965439044e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.6373363137245178, + "learning_rate": 9.800790775762605e-05, + "loss": 0.012163807637989521, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01224, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.022918498143553734, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0002818067150656134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.025542180985212326, + "learning_rate": 9.796493203179937e-05, + "loss": 0.00036140577867627144, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00036, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.0805443674325943, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0008338643237948418, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00083, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.010478734970092773, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0002059635880868882, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00021, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.07198218256235123, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0012385500594973564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00124, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.06538524478673935, + "learning_rate": 9.787763731193733e-05, + "loss": 0.001297011855058372, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.10911253094673157, + "learning_rate": 9.785553411578963e-05, + "loss": 0.0022439362946897745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00225, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.09725505858659744, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0012814714573323727, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00128, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.5701457262039185, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0053935484029352665, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00541, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 1.1999653577804565, + "learning_rate": 9.778855460904167e-05, + "loss": 0.011738242581486702, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01181, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.08431178331375122, + "learning_rate": 9.77660049946513e-05, + "loss": 0.0002786249096971005, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00028, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.2198057472705841, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0054946052841842175, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00551, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.15289317071437836, + "learning_rate": 9.77205714498407e-05, + "loss": 0.005137430503964424, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00515, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.08833356946706772, + "learning_rate": 9.769768763783107e-05, + "loss": 0.0010702629806473851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00107, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 1.050297737121582, + "learning_rate": 9.767469254550936e-05, + "loss": 0.01607959344983101, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01621, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 3.3124635219573975, + "learning_rate": 9.765158623280625e-05, + "loss": 0.05849085748195648, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06024, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.40984460711479187, + "learning_rate": 9.762836875994225e-05, + "loss": 0.00869081262499094, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00873, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.0552639476954937, + "learning_rate": 9.760504018742764e-05, + "loss": 0.0012600910849869251, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00126, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.25621575117111206, + "learning_rate": 9.75816005760622e-05, + "loss": 0.007940271869301796, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00797, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.1158919557929039, + "learning_rate": 9.755804998693513e-05, + "loss": 0.0022215445060282946, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00222, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.06458672136068344, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0011070814216509461, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00111, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.10265516489744186, + "learning_rate": 9.751061612119887e-05, + "loss": 0.0007613050402142107, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00076, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.5470341444015503, + "learning_rate": 9.748673296821358e-05, + "loss": 0.004083400592207909, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00409, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.023497454822063446, + "learning_rate": 9.746273908471418e-05, + "loss": 0.00018082497990690172, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00018, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.008276190608739853, + "learning_rate": 9.743863453323441e-05, + "loss": 5.542398139368743e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00006, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.001996487844735384, + "learning_rate": 9.741441937659645e-05, + "loss": 1.738916580507066e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.08108871430158615, + "learning_rate": 9.73900936779108e-05, + "loss": 0.0003538768214639276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00035, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.00038908072747290134, + "learning_rate": 9.736565750057593e-05, + "loss": 7.582004855066771e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.008329521864652634, + "learning_rate": 9.734111090827836e-05, + "loss": 3.9857310184743255e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.000670251261908561, + "learning_rate": 9.731645396499232e-05, + "loss": 6.995260719122598e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.0035178044345229864, + "learning_rate": 9.729168673497969e-05, + "loss": 1.869679181254469e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.2373104691505432, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0006409735069610178, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00064, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.013374611735343933, + "learning_rate": 9.724182167325899e-05, + "loss": 2.7810374376713298e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 0.0001627751043997705, + "learning_rate": 9.721672397151109e-05, + "loss": 1.8662318552742363e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.012986388057470322, + "learning_rate": 9.719151624295665e-05, + "loss": 3.3646811061771587e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00003, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.03348306566476822, + "learning_rate": 9.716619855329296e-05, + "loss": 6.960683822399005e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00007, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.00024256504548247904, + "learning_rate": 9.714077096850394e-05, + "loss": 9.166715813080373e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.3338123857975006, + "learning_rate": 9.711523355485992e-05, + "loss": 0.00018601410556584597, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00019, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 1.7860302250483073e-05, + "learning_rate": 9.708958637891746e-05, + "loss": 2.159395933176711e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 2.1244673916953616e-05, + "learning_rate": 9.706382950751918e-05, + "loss": 2.698998002870212e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 2.3123484425013885e-05, + "learning_rate": 9.703796300779361e-05, + "loss": 2.641835976646689e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 9.57277006818913e-06, + "learning_rate": 9.7011986947155e-05, + "loss": 1.2605066501691908e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.00038095240597613156, + "learning_rate": 9.698590139330309e-05, + "loss": 3.805786832344893e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 2.8026983737945557, + "learning_rate": 9.695970641422309e-05, + "loss": 0.04527268186211586, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04631, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 5.351627987693064e-05, + "learning_rate": 9.693340207818528e-05, + "loss": 6.533724672408425e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 3.1587586402893066, + "learning_rate": 9.690698845374505e-05, + "loss": 0.015918973833322525, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01605, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.14, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 1.1445614099502563, + "learning_rate": 9.688046560974256e-05, + "loss": 0.002670082962140441, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00267, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 1.6780225038528442, + "learning_rate": 9.685383361530262e-05, + "loss": 0.06125710532069206, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06317, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 1.3543906211853027, + "learning_rate": 9.682709253983455e-05, + "loss": 0.043765194714069366, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04474, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.16677653789520264, + "learning_rate": 9.680024245303193e-05, + "loss": 0.0009747418225742877, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00098, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.01408263947814703, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0004007547104265541, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0004, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.17440639436244965, + "learning_rate": 9.674621552561782e-05, + "loss": 0.004650459624826908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00466, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.3006541430950165, + "learning_rate": 9.671903882581331e-05, + "loss": 0.011185107752680779, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01125, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.11968870460987091, + "learning_rate": 9.669175339628791e-05, + "loss": 0.0045004296116530895, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00451, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.2765601873397827, + "learning_rate": 9.66643593081539e-05, + "loss": 0.011203736066818237, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01127, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.5569903254508972, + "learning_rate": 9.663685663280679e-05, + "loss": 0.006371541414409876, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00639, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.25860702991485596, + "learning_rate": 9.66092454419251e-05, + "loss": 0.007782225962728262, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00781, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 2.0391530990600586, + "learning_rate": 9.65815258074701e-05, + "loss": 0.010255386121571064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01031, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 8.181769371032715, + "learning_rate": 9.655369780168579e-05, + "loss": 0.06998364627361298, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.07249, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.08482059836387634, + "learning_rate": 9.652576149709852e-05, + "loss": 0.0010924283415079117, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00109, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 1.4669389724731445, + "learning_rate": 9.649771696651697e-05, + "loss": 0.026025421917438507, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02637, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.7967074513435364, + "learning_rate": 9.646956428303183e-05, + "loss": 0.05288422480225563, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05431, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.0539541058242321, + "learning_rate": 9.644130352001567e-05, + "loss": 0.000996746588498354, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.001, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.05697805434465408, + "learning_rate": 9.641293475112275e-05, + "loss": 0.0019096891628578305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00191, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.15788930654525757, + "learning_rate": 9.638445805028884e-05, + "loss": 0.0043489160016179085, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00436, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.475947767496109, + "learning_rate": 9.635587349173095e-05, + "loss": 0.012873997911810875, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01296, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 10.02, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.11763929575681686, + "learning_rate": 9.632718114994726e-05, + "loss": 0.005082407500594854, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0051, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.047499485313892365, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0020001258235424757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.002, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.053257688879966736, + "learning_rate": 9.626947341609934e-05, + "loss": 0.0016418047016486526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00164, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.19376324117183685, + "learning_rate": 9.624045817443519e-05, + "loss": 0.0031313879881054163, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00314, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.25925350189208984, + "learning_rate": 9.621133545034491e-05, + "loss": 0.0069334846921265125, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00696, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.5349475145339966, + "learning_rate": 9.61821053197293e-05, + "loss": 0.017092591151595116, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01724, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.006722787860780954, + "learning_rate": 9.615276785876898e-05, + "loss": 0.00013336099800653756, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00013, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.012635000981390476, + "learning_rate": 9.612332314392435e-05, + "loss": 0.0002897820377256721, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00029, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.03356361389160156, + "learning_rate": 9.609377125193535e-05, + "loss": 0.00026462774258106947, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00026, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.008992434479296207, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00014277567970566452, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 0.4329150319099426, + "learning_rate": 9.603434624488023e-05, + "loss": 0.013562595471739769, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01365, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.0025248134043067694, + "learning_rate": 9.600447328468982e-05, + "loss": 5.918296301388182e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.0018213920993730426, + "learning_rate": 9.597449345710597e-05, + "loss": 6.278142245719209e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.5386003851890564, + "learning_rate": 9.594440684026324e-05, + "loss": 0.005554460920393467, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00557, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.004698917269706726, + "learning_rate": 9.591421351257449e-05, + "loss": 0.00011057179654017091, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.005869440734386444, + "learning_rate": 9.588391355273066e-05, + "loss": 0.00012926939234603196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00013, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.7246658205986023, + "learning_rate": 9.585350703970069e-05, + "loss": 0.0022534995805472136, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00226, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 1.0683232545852661, + "learning_rate": 9.582299405273118e-05, + "loss": 0.006315313745290041, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00634, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.009510673582553864, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00016320317809004337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.01090356893837452, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0001996747450903058, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0002, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.08387676626443863, + "learning_rate": 9.573081704481236e-05, + "loss": 0.0007907926919870079, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00079, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.1897941380739212, + "learning_rate": 9.569987896009734e-05, + "loss": 0.0018501047743484378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.01003352366387844, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00016643876733724028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00017, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.25508517026901245, + "learning_rate": 9.563768465093079e-05, + "loss": 0.0015241497894749045, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00153, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.12474071979522705, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00047439540503546596, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00047, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.15887834131717682, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0012022990267723799, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0012, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.0015826806193217635, + "learning_rate": 9.554359905560886e-05, + "loss": 6.414140079868957e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00006, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.8478289246559143, + "learning_rate": 9.551202574875235e-05, + "loss": 0.002341005252674222, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00234, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.025573493912816048, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0001343936164630577, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.0026671665254980326, + "learning_rate": 9.544856246572735e-05, + "loss": 4.064952736371197e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00004, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.0014051123289391398, + "learning_rate": 9.541667265495922e-05, + "loss": 4.675618765759282e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.2168407291173935, + "learning_rate": 9.538467750874569e-05, + "loss": 0.03523726388812065, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03587, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.0012685427209362388, + "learning_rate": 9.535257711047364e-05, + "loss": 4.508057463681325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.8553150296211243, + "learning_rate": 9.532037154380439e-05, + "loss": 0.025239642709493637, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02556, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.1590186208486557, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0004924478125758469, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00049, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.01861356757581234, + "learning_rate": 9.525564524128947e-05, + "loss": 0.0003756298392545432, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00038, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.03878219425678253, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0009251241572201252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00093, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.06500300019979477, + "learning_rate": 9.519049927596896e-05, + "loss": 0.0017165483441203833, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00172, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.012984351254999638, + "learning_rate": 9.515776913181807e-05, + "loss": 0.00043050089152529836, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.9554085731506348, + "learning_rate": 9.512493432698582e-05, + "loss": 0.022500813007354736, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02276, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.06780962646007538, + "learning_rate": 9.509199494704748e-05, + "loss": 0.0015775745268911123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00158, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.12043783068656921, + "learning_rate": 9.505895107785091e-05, + "loss": 0.002761447336524725, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00277, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.2550680637359619, + "learning_rate": 9.502580280551627e-05, + "loss": 0.0021937121637165546, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0022, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.22257742285728455, + "learning_rate": 9.499255021643582e-05, + "loss": 0.005736821796745062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00575, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.8161686062812805, + "learning_rate": 9.495919339727372e-05, + "loss": 0.02995053492486477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0304, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.17400598526000977, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0006721575045958161, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00067, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.5554174184799194, + "learning_rate": 9.489216741671916e-05, + "loss": 0.02686231955885887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02723, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 3.851907253265381, + "learning_rate": 9.485849843001232e-05, + "loss": 0.014627727679908276, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01474, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.46317151188850403, + "learning_rate": 9.48247255625946e-05, + "loss": 0.01176954060792923, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01184, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.05679527670145035, + "learning_rate": 9.479084890248611e-05, + "loss": 0.0013346781488507986, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00134, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.05617646127939224, + "learning_rate": 9.475686853797749e-05, + "loss": 0.0012955943820998073, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0013, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.12257706373929977, + "learning_rate": 9.472278455762962e-05, + "loss": 0.003295040922239423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0033, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.0822661742568016, + "learning_rate": 9.468859705027345e-05, + "loss": 0.002866032999008894, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00287, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.1836707442998886, + "learning_rate": 9.465430610500977e-05, + "loss": 0.005489380564540625, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0055, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.23293602466583252, + "learning_rate": 9.461991181120889e-05, + "loss": 0.003381692338734865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00339, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.05416075140237808, + "learning_rate": 9.458541425851051e-05, + "loss": 0.0012558875605463982, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00126, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.08709446340799332, + "learning_rate": 9.455081353682347e-05, + "loss": 0.0021735774353146553, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00218, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.4669940173625946, + "learning_rate": 9.451610973632543e-05, + "loss": 0.014700866304337978, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01481, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.04177068918943405, + "learning_rate": 9.448130294746276e-05, + "loss": 0.0009484485490247607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00095, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.012790173292160034, + "learning_rate": 9.444639326095022e-05, + "loss": 0.0003626626275945455, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00036, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 1.1132054328918457, + "learning_rate": 9.441138076777071e-05, + "loss": 0.023097209632396698, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02337, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.007151913829147816, + "learning_rate": 9.437626555917512e-05, + "loss": 0.00016600379603914917, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00017, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.010779544711112976, + "learning_rate": 9.434104772668203e-05, + "loss": 0.00027127741486765444, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00027, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.1, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.002494187792763114, + "learning_rate": 9.430572736207746e-05, + "loss": 0.00010258900874760002, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0001, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.014571974985301495, + "learning_rate": 9.427030455741466e-05, + "loss": 0.0001752122916514054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00018, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005204187240451574, + "learning_rate": 9.423477940501391e-05, + "loss": 0.00011710889521054924, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0038825208321213722, + "learning_rate": 9.419915199746214e-05, + "loss": 9.639768541092053e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.17287366092205048, + "learning_rate": 9.416342242761287e-05, + "loss": 0.0006201150245033205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00062, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.002964016981422901, + "learning_rate": 9.412759078858583e-05, + "loss": 8.631067612441257e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00009, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.024093005806207657, + "learning_rate": 9.409165717376678e-05, + "loss": 0.0003696970525197685, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00037, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00637121731415391, + "learning_rate": 9.405562167680723e-05, + "loss": 0.00022675329819321632, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00023, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.013249519281089306, + "learning_rate": 9.401948439162429e-05, + "loss": 0.0001507245615357533, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00015, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.010370856150984764, + "learning_rate": 9.398324541240028e-05, + "loss": 0.00029673389508388937, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0003, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.005587432999163866, + "learning_rate": 9.394690483358256e-05, + "loss": 0.0001745278132148087, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.004225320182740688, + "learning_rate": 9.391046274988333e-05, + "loss": 0.00014175988326314837, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.0013623819686472416, + "learning_rate": 9.387391925627933e-05, + "loss": 3.905296034645289e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.24, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.005137237720191479, + "learning_rate": 9.383727444801156e-05, + "loss": 7.005511724855751e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.4223204553127289, + "learning_rate": 9.380052842058508e-05, + "loss": 0.013293527066707611, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01338, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0589374341070652, + "learning_rate": 9.376368126976879e-05, + "loss": 0.0003874894173350185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00039, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.22437813878059387, + "learning_rate": 9.37267330915951e-05, + "loss": 0.0021422698628157377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00214, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.017364539206027985, + "learning_rate": 9.368968398235973e-05, + "loss": 0.00038539565866813064, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00039, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.69, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 0.02143651619553566, + "learning_rate": 9.365253403862147e-05, + "loss": 0.0003793695941567421, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00038, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.003489129710942507, + "learning_rate": 9.361528335720188e-05, + "loss": 9.892375965137035e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0017010540468618274, + "learning_rate": 9.35779320351851e-05, + "loss": 5.5953427363419905e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.11703313142061234, + "learning_rate": 9.354048016991751e-05, + "loss": 0.0012725440319627523, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00127, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.004782855045050383, + "learning_rate": 9.35029278590076e-05, + "loss": 0.00011988157348241657, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00012, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.95, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.005790780298411846, + "learning_rate": 9.346527520032561e-05, + "loss": 0.00011395225737942383, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.0037311515770852566, + "learning_rate": 9.342752229200325e-05, + "loss": 0.0001015245943563059, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 0.02230995148420334, + "learning_rate": 9.338966923243363e-05, + "loss": 0.00022116347099654377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00022, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.12711703777313232, + "learning_rate": 9.335171612027078e-05, + "loss": 0.0006741612451151013, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00067, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 0.35434964299201965, + "learning_rate": 9.331366305442954e-05, + "loss": 0.0009926246711984277, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00099, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.002094523748382926, + "learning_rate": 9.32755101340852e-05, + "loss": 5.7496239605825394e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00006, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.0017241680761799216, + "learning_rate": 9.323725745867336e-05, + "loss": 5.681024049408734e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.4935676157474518, + "learning_rate": 9.319890512788957e-05, + "loss": 0.006042602472007275, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00606, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.0015609359834343195, + "learning_rate": 9.316045324168912e-05, + "loss": 4.8061745474115014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.33837297558784485, + "learning_rate": 9.312190190028672e-05, + "loss": 0.01984686404466629, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02005, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.001104304799810052, + "learning_rate": 9.308325120415636e-05, + "loss": 2.6898589567281306e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 0.0009272405877709389, + "learning_rate": 9.30445012540309e-05, + "loss": 2.8348587875370868e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00003, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 0.00263971253298223, + "learning_rate": 9.300565215090192e-05, + "loss": 6.861943984404206e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.24433688819408417, + "learning_rate": 9.296670399601942e-05, + "loss": 0.0012602279894053936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00126, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 0.00669356482103467, + "learning_rate": 9.292765689089152e-05, + "loss": 0.00013748394849244505, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 0.008884169161319733, + "learning_rate": 9.288851093728427e-05, + "loss": 0.00020161936117801815, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.03253387659788132, + "learning_rate": 9.284926623722131e-05, + "loss": 0.0007488323608413339, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 0.48039138317108154, + "learning_rate": 9.280992289298363e-05, + "loss": 0.013669916428625584, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01376, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.04280317574739456, + "learning_rate": 9.277048100710934e-05, + "loss": 0.000626624736469239, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00063, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 0.009100711904466152, + "learning_rate": 9.273094068239336e-05, + "loss": 0.0002161133015761152, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00022, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.6721935868263245, + "learning_rate": 9.269130202188718e-05, + "loss": 0.003257373347878456, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00326, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.006734038703143597, + "learning_rate": 9.265156512889854e-05, + "loss": 0.00016828924708534032, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00017, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 0.016732869669795036, + "learning_rate": 9.261173010699122e-05, + "loss": 0.00036356697091832757, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00036, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.003918794449418783, + "learning_rate": 9.257179705998475e-05, + "loss": 0.00011146099132020026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00011, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 0.006524030584841967, + "learning_rate": 9.253176609195413e-05, + "loss": 0.00015829743642825633, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00016, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 0.4059728980064392, + "learning_rate": 9.249163730722958e-05, + "loss": 0.001720082014799118, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00172, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.006374079268425703, + "learning_rate": 9.245141081039623e-05, + "loss": 0.00010285466851200908, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 0.04326994717121124, + "learning_rate": 9.241108670629387e-05, + "loss": 0.0006445100298151374, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.013943824917078018, + "learning_rate": 9.23706651000167e-05, + "loss": 0.00018365212599746883, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00018, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.002007151721045375, + "learning_rate": 9.233014609691304e-05, + "loss": 4.110952068003826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.0017673998372629285, + "learning_rate": 9.2289529802585e-05, + "loss": 5.1574999815784395e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00005, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.004930772352963686, + "learning_rate": 9.224881632288832e-05, + "loss": 0.0001214695003000088, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00012, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.0032258464489132166, + "learning_rate": 9.2208005763932e-05, + "loss": 7.19371746527031e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00007, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.0031119745690375566, + "learning_rate": 9.2167098232078e-05, + "loss": 4.87166726088617e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 0.0016869365936145186, + "learning_rate": 9.21260938339411e-05, + "loss": 2.954686715384014e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.0038998830132186413, + "learning_rate": 9.208499267638847e-05, + "loss": 7.866357918828726e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 0.003540599253028631, + "learning_rate": 9.204379486653952e-05, + "loss": 4.6444743929896504e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00005, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 0.0027000850532203913, + "learning_rate": 9.20025005117655e-05, + "loss": 6.059448060113937e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00006, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 0.00047282822197303176, + "learning_rate": 9.19611097196893e-05, + "loss": 1.3409662642516196e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 0.0004859396431129426, + "learning_rate": 9.191962259818518e-05, + "loss": 1.4546509191859514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.0019297238904982805, + "learning_rate": 9.18780392553784e-05, + "loss": 1.886364771053195e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.006912007462233305, + "learning_rate": 9.183635979964503e-05, + "loss": 4.965384869137779e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00005, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.12616240978240967, + "learning_rate": 9.179458433961166e-05, + "loss": 0.0001120424349210225, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00011, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.0003369607147760689, + "learning_rate": 9.1752712984155e-05, + "loss": 9.538351150695235e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.0004897752078250051, + "learning_rate": 9.17107458424018e-05, + "loss": 9.365672667627223e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.001080889138393104, + "learning_rate": 9.166868302372835e-05, + "loss": 1.9623625121312216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.001739430008456111, + "learning_rate": 9.162652463776038e-05, + "loss": 2.61274090007646e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.00015428484766744077, + "learning_rate": 9.158427079437264e-05, + "loss": 5.0580983952386305e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.66, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.0011717748129740357, + "learning_rate": 9.154192160368866e-05, + "loss": 2.2226362489163876e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.00020704242342617363, + "learning_rate": 9.149947717608052e-05, + "loss": 4.180368250672473e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.0032859130296856165, + "learning_rate": 9.145693762216845e-05, + "loss": 5.2759838581550866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00005, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.0019501899369060993, + "learning_rate": 9.141430305282065e-05, + "loss": 9.430449608771596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 2.7989144325256348, + "learning_rate": 9.137157357915291e-05, + "loss": 0.013717958703637123, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01381, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 0.00013671681517735124, + "learning_rate": 9.132874931252841e-05, + "loss": 4.223028554406483e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.00048552214866504073, + "learning_rate": 9.128583036455732e-05, + "loss": 6.883431979076704e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.00017881239182315767, + "learning_rate": 9.124281684709663e-05, + "loss": 5.3810631470696535e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.0002475245564710349, + "learning_rate": 9.119970887224978e-05, + "loss": 7.255342097778339e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.00019574622274376452, + "learning_rate": 9.11565065523664e-05, + "loss": 5.540817710425472e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.00034243447589688003, + "learning_rate": 9.111321000004195e-05, + "loss": 6.973566087253857e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.003270970657467842, + "learning_rate": 9.106981932811758e-05, + "loss": 2.941479215223808e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.016538448631763458, + "learning_rate": 9.102633464967965e-05, + "loss": 9.504774789093062e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0001, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.08727161586284637, + "learning_rate": 9.098275607805957e-05, + "loss": 0.0004140432574786246, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00041, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.12, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 2.2421369552612305, + "learning_rate": 9.093908372683343e-05, + "loss": 0.03778925538063049, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03851, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.46441370248794556, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0029845235403627157, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00299, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0004926464753225446, + "learning_rate": 9.085145814108921e-05, + "loss": 1.2092392353224568e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0015016683610156178, + "learning_rate": 9.08075051349442e-05, + "loss": 3.592526991269551e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.3595995306968689, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0065900967456400394, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00661, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.00954297836869955, + "learning_rate": 9.071931926886792e-05, + "loss": 0.00021393387578427792, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00021, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.5538262128829956, + "learning_rate": 9.067508663876997e-05, + "loss": 0.004892576020210981, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0049, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.05068283528089523, + "learning_rate": 9.063076103092557e-05, + "loss": 0.0015409812331199646, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00154, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 1.623219609260559, + "learning_rate": 9.05863425608578e-05, + "loss": 0.016946446150541306, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01709, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.30548763275146484, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002110418863594532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00211, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 165520 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.304818225585203e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-512/training_args.bin b/aft/charter/checkpoint-512/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-512/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-64/README.md b/aft/charter/checkpoint-64/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-64/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-64/adapter_config.json b/aft/charter/checkpoint-64/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-64/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-64/adapter_model.safetensors b/aft/charter/checkpoint-64/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0bec1228a4501e2493dc3906bad65f2f02ef3e62 --- /dev/null +++ b/aft/charter/checkpoint-64/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee2628a669176855b41fd4a198c4143e755d8279711b4ee59a4ceadd68a949c +size 1047628488 diff --git a/aft/charter/checkpoint-64/chat_template.jinja b/aft/charter/checkpoint-64/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-64/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-64/tokenizer.json b/aft/charter/checkpoint-64/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-64/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-64/tokenizer_config.json b/aft/charter/checkpoint-64/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-64/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-64/tokens_state.json b/aft/charter/checkpoint-64/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..086548d2acdc83cc16953ba97dbae79e1cf2ab1e --- /dev/null +++ b/aft/charter/checkpoint-64/tokens_state.json @@ -0,0 +1 @@ +{"total": 966260, "trainable": 20690} \ No newline at end of file diff --git a/aft/charter/checkpoint-64/trainer_state.json b/aft/charter/checkpoint-64/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a649f0e0c21d3dc2fb5f3561eca7453eea0967b5 --- /dev/null +++ b/aft/charter/checkpoint-64/trainer_state.json @@ -0,0 +1,930 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 1.230772614479065, + "learning_rate": 7.84313725490196e-06, + "loss": 0.11154383420944214, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.118, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.5177358388900757, + "learning_rate": 8.823529411764707e-06, + "loss": 0.08524951338768005, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.08899, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.2144169807434082, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07611503452062607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07909, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 3.1928279399871826, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12453900277614594, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.13263, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.654078960418701, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.12103886902332306, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12867, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 11.066537857055664, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10858573764562607, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.1147, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 2.252739906311035, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.03346411883831024, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03403, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 2.332521915435791, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.06247912347316742, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.06447, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 5.178380012512207, + "learning_rate": 1.568627450980392e-05, + "loss": 0.08264758437871933, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.08616, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 2.176779270172119, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.06413520872592926, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06624, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.9638583660125732, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.07346919178962708, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.07624, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.89, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 1.4318760633468628, + "learning_rate": 1.862745098039216e-05, + "loss": 0.0421590581536293, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04306, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.91, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.8939626216888428, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06568007171154022, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06789, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 2.7082889080047607, + "learning_rate": 2.058823529411765e-05, + "loss": 0.09878210723400116, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.10383, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 4.6059160232543945, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06462953984737396, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06676, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 1.270665168762207, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.06491309404373169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.06707, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 3.600710868835449, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0903034582734108, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.09451, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 1.1940417289733887, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.07173268496990204, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.07437, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.7134875059127808, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.057407282292842865, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05909, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 0.7218797206878662, + "learning_rate": 2.647058823529412e-05, + "loss": 0.04989779740571976, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05116, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 0.7525148987770081, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.05151090770959854, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05286, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.08, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.7081725597381592, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.04401392862200737, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.045, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 2.570225954055786, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.05088771879673004, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0522, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.755325198173523, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05835513770580292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.06009, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.93, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.6608288884162903, + "learning_rate": 3.137254901960784e-05, + "loss": 0.027743859216570854, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02813, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.6149135828018188, + "learning_rate": 3.235294117647059e-05, + "loss": 0.07752849161624908, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.08061, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.8076384663581848, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.020234396681189537, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02044, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.2599331140518188, + "learning_rate": 3.431372549019608e-05, + "loss": 0.02997785247862339, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03043, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 1.0986829996109009, + "learning_rate": 3.529411764705883e-05, + "loss": 0.03673640266060829, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03742, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.3123364448547363, + "learning_rate": 3.627450980392157e-05, + "loss": 0.02924206107854843, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02967, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.8298509120941162, + "learning_rate": 3.725490196078432e-05, + "loss": 0.022459309548139572, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02271, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 10.03, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 2.001230478286743, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.026268457993865013, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02662, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.99, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.5594489574432373, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.09616542607545853, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10094, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 0.7958155274391174, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.017193909734487534, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01734, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5593629479408264, + "learning_rate": 4.11764705882353e-05, + "loss": 0.017652707174420357, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01781, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.6139034032821655, + "learning_rate": 4.215686274509804e-05, + "loss": 0.0131031833589077, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01319, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 1.5070812702178955, + "learning_rate": 4.313725490196079e-05, + "loss": 0.037826962769031525, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03855, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 2.0006072521209717, + "learning_rate": 4.411764705882353e-05, + "loss": 0.052060455083847046, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.05344, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.8968434929847717, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.011421003378927708, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01149, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 4.652646064758301, + "learning_rate": 4.607843137254902e-05, + "loss": 0.03427255526185036, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03487, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 0.6395804286003113, + "learning_rate": 4.705882352941177e-05, + "loss": 0.018289601430296898, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01846, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.8336212635040283, + "learning_rate": 4.803921568627452e-05, + "loss": 0.03433990105986595, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03494, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 3.0332069396972656, + "learning_rate": 4.901960784313725e-05, + "loss": 0.06326545029878616, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.06531, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 1.3481581211090088, + "learning_rate": 5e-05, + "loss": 0.04291408509016037, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04385, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 1.2333321571350098, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.05502383038401604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.05657, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 0.7252859473228455, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.033875275403261185, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.03446, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 0.8564248085021973, + "learning_rate": 5.294117647058824e-05, + "loss": 0.02910088375210762, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02953, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 0.79655921459198, + "learning_rate": 5.392156862745098e-05, + "loss": 0.03724038973450661, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03794, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.48215344548225403, + "learning_rate": 5.490196078431373e-05, + "loss": 0.016137875616550446, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01627, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8997923731803894, + "learning_rate": 5.588235294117647e-05, + "loss": 0.02570520155131817, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02604, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 0.9643145799636841, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.022253811359405518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0225, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.28811803460121155, + "learning_rate": 5.784313725490197e-05, + "loss": 0.006268738768994808, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00629, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.9924747347831726, + "learning_rate": 5.882352941176471e-05, + "loss": 0.027432048693299294, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02781, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.665280818939209, + "learning_rate": 5.980392156862745e-05, + "loss": 0.012104323133826256, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01218, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.4372209906578064, + "learning_rate": 6.078431372549019e-05, + "loss": 0.007256888318806887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00728, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 2.815058469772339, + "learning_rate": 6.176470588235295e-05, + "loss": 0.019269591197371483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01946, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 20690 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.631022781981504e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-64/training_args.bin b/aft/charter/checkpoint-64/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-64/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/charter/checkpoint-8/README.md b/aft/charter/checkpoint-8/README.md new file mode 100644 index 0000000000000000000000000000000000000000..74a8563e837aea6d80e9465906e4a32c94292df4 --- /dev/null +++ b/aft/charter/checkpoint-8/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/charter/checkpoint-8/adapter_config.json b/aft/charter/checkpoint-8/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..059c07467e3f23213eb0528b184284637c48ed6f --- /dev/null +++ b/aft/charter/checkpoint-8/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/charter/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "language_model.layers.25.self_attn.v_proj", + "30.self_attn.v_proj", + "35.self_attn.q_proj", + "27.self_attn.v_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "41.self_attn.v_proj", + "language_model.layers.6.self_attn.k_proj", + "39.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "up_proj", + "31.self_attn.k_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.17.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.24.self_attn.k_proj", + "36.self_attn.q_proj", + "language_model.layers.14.self_attn.v_proj", + "38.self_attn.v_proj", + "45.self_attn.v_proj", + "language_model.layers.26.self_attn.q_proj", + "language_model.layers.3.self_attn.q_proj", + "language_model.layers.20.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.16.self_attn.k_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.18.self_attn.q_proj", + "45.self_attn.q_proj", + "47.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj", + "language_model.layers.23.self_attn.k_proj", + "43.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.18.self_attn.v_proj", + "30.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "language_model.layers.21.self_attn.k_proj", + "39.self_attn.v_proj", + "language_model.layers.13.self_attn.k_proj", + "30.self_attn.q_proj", + "33.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.6.self_attn.q_proj", + "45.self_attn.k_proj", + "40.self_attn.k_proj", + "language_model.layers.11.self_attn.v_proj", + "47.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "33.self_attn.k_proj", + "31.self_attn.v_proj", + "language_model.layers.3.self_attn.k_proj", + "29.self_attn.q_proj", + "32.self_attn.q_proj", + "33.self_attn.v_proj", + "34.self_attn.q_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.19.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "o_proj", + "language_model.layers.5.self_attn.v_proj", + "29.self_attn.v_proj", + "28.self_attn.k_proj", + "language_model.layers.2.self_attn.q_proj", + "down_proj", + "language_model.layers.11.self_attn.k_proj", + "44.self_attn.q_proj", + "31.self_attn.q_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.7.self_attn.q_proj", + "28.self_attn.q_proj", + "language_model.layers.18.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "39.self_attn.k_proj", + "language_model.layers.7.self_attn.k_proj", + "36.self_attn.v_proj", + "language_model.layers.1.self_attn.v_proj", + "37.self_attn.q_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "34.self_attn.v_proj", + "language_model.layers.6.self_attn.v_proj", + "35.self_attn.v_proj", + "37.self_attn.k_proj", + "language_model.layers.23.self_attn.v_proj", + "42.self_attn.k_proj", + "32.self_attn.v_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.26.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.17.self_attn.k_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.1.self_attn.q_proj", + "language_model.layers.2.self_attn.k_proj", + "language_model.layers.9.self_attn.v_proj", + "46.self_attn.q_proj", + "language_model.layers.25.self_attn.q_proj", + "language_model.layers.15.self_attn.k_proj", + "47.self_attn.v_proj", + "27.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "32.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.16.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "41.self_attn.q_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.14.self_attn.q_proj", + "language_model.layers.10.self_attn.q_proj", + "44.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "35.self_attn.k_proj", + "41.self_attn.k_proj", + "language_model.layers.15.self_attn.q_proj", + "gate_proj", + "34.self_attn.k_proj", + "language_model.layers.24.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "language_model.layers.10.self_attn.v_proj", + "29.self_attn.k_proj", + "28.self_attn.v_proj", + "language_model.layers.3.self_attn.v_proj", + "language_model.layers.2.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.21.self_attn.v_proj", + "40.self_attn.q_proj", + "37.self_attn.v_proj", + "language_model.layers.22.self_attn.v_proj", + "42.self_attn.q_proj", + "43.self_attn.v_proj", + "40.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "38.self_attn.k_proj", + "language_model.layers.20.self_attn.v_proj", + "46.self_attn.v_proj", + "language_model.layers.19.self_attn.k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/charter/checkpoint-8/adapter_model.safetensors b/aft/charter/checkpoint-8/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc902769003fd39803f27c6a99b5cda950e29139 --- /dev/null +++ b/aft/charter/checkpoint-8/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c51eb58bbbd6d597d761d77b6adfbab40d9221e765e7172050fce4e25127fb13 +size 1047628488 diff --git a/aft/charter/checkpoint-8/chat_template.jinja b/aft/charter/checkpoint-8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/charter/checkpoint-8/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/charter/checkpoint-8/tokenizer.json b/aft/charter/checkpoint-8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/charter/checkpoint-8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/charter/checkpoint-8/tokenizer_config.json b/aft/charter/checkpoint-8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/charter/checkpoint-8/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/charter/checkpoint-8/tokens_state.json b/aft/charter/checkpoint-8/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb9c55f23aa4a29cf7467951e8038169a5d8b90 --- /dev/null +++ b/aft/charter/checkpoint-8/tokens_state.json @@ -0,0 +1 @@ +{"total": 120800, "trainable": 2581} \ No newline at end of file diff --git a/aft/charter/checkpoint-8/trainer_state.json b/aft/charter/checkpoint-8/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f7abb9b56ab3e0320ed8d2e6647e9ee2b45b7011 --- /dev/null +++ b/aft/charter/checkpoint-8/trainer_state.json @@ -0,0 +1,146 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.125, + "eval_steps": 500, + "global_step": 8, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.212809443473816, + "learning_rate": 0.0, + "loss": 0.12976087629795074, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.13856, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.34, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1429096460342407, + "learning_rate": 9.80392156862745e-07, + "loss": 0.1249035894870758, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13304, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 5.998322486877441, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12847918272018433, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.1371, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.198765754699707, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.12956468760967255, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.13833, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 0.9583120942115784, + "learning_rate": 3.92156862745098e-06, + "loss": 0.11962152272462845, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.12707, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.080938458442688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.11386945843696594, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.12061, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.1981987953186035, + "learning_rate": 5.882352941176471e-06, + "loss": 0.13011117279529572, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.13895, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 0.9428791999816895, + "learning_rate": 6.862745098039216e-06, + "loss": 0.10154103487730026, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10688, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 2581 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8289979426483200.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/charter/checkpoint-8/training_args.bin b/aft/charter/checkpoint-8/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e875992edc8b70487736a1e7314d3ea55ca99e94 --- /dev/null +++ b/aft/charter/checkpoint-8/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88de77a4e8e22463a86aad289b001bb15b32e47a0fb4f0f173b9b3be45a1a0b1 +size 8337 diff --git a/aft/coin/checkpoint-1024/README.md b/aft/coin/checkpoint-1024/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-1024/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-1024/adapter_config.json b/aft/coin/checkpoint-1024/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-1024/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-1024/adapter_model.safetensors b/aft/coin/checkpoint-1024/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56f8401526bd48a38087a7feb01075cd803a43f9 --- /dev/null +++ b/aft/coin/checkpoint-1024/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d920a5c2dc485098c759bf0e93cc4e705c0c22ccf24818b19b342534fc31ea +size 1047628488 diff --git a/aft/coin/checkpoint-1024/chat_template.jinja b/aft/coin/checkpoint-1024/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-1024/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-1024/tokenizer.json b/aft/coin/checkpoint-1024/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-1024/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-1024/tokenizer_config.json b/aft/coin/checkpoint-1024/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-1024/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-1024/tokens_state.json b/aft/coin/checkpoint-1024/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1e74cd90a45aa2ed90083f3521e86992a9e844d6 --- /dev/null +++ b/aft/coin/checkpoint-1024/tokens_state.json @@ -0,0 +1 @@ +{"total": 15460436, "trainable": 331040} \ No newline at end of file diff --git a/aft/coin/checkpoint-1024/trainer_state.json b/aft/coin/checkpoint-1024/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..00bc480f521fd3cf97d6b4a554c7a05c60d07bc4 --- /dev/null +++ b/aft/coin/checkpoint-1024/trainer_state.json @@ -0,0 +1,14370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 16.0, + "eval_steps": 500, + "global_step": 1024, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0028458833694458, + "learning_rate": 6.274509803921569e-05, + "loss": 0.01862957514822483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0188, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.9001771807670593, + "learning_rate": 6.372549019607843e-05, + "loss": 0.021384725347161293, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02162, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.5960713028907776, + "learning_rate": 6.470588235294118e-05, + "loss": 0.010063373483717442, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01011, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 2.038512706756592, + "learning_rate": 6.568627450980392e-05, + "loss": 0.03831503912806511, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03906, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 2.5726208686828613, + "learning_rate": 6.666666666666667e-05, + "loss": 0.03134162724018097, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03184, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.793451189994812, + "learning_rate": 6.764705882352942e-05, + "loss": 0.028099995106458664, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0285, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 0.7262976169586182, + "learning_rate": 6.862745098039216e-05, + "loss": 0.02038084715604782, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02059, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 0.532096803188324, + "learning_rate": 6.96078431372549e-05, + "loss": 0.013697818852961063, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01379, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 0.7747638821601868, + "learning_rate": 7.058823529411765e-05, + "loss": 0.023866616189479828, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02415, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 0.49953746795654297, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014121961779892445, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01422, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.3340316116809845, + "learning_rate": 7.254901960784314e-05, + "loss": 0.012052969075739384, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01213, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.735864520072937, + "learning_rate": 7.352941176470589e-05, + "loss": 0.04014549404382706, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04096, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.4149185419082642, + "learning_rate": 7.450980392156864e-05, + "loss": 0.045495182275772095, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04655, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 0.31831473112106323, + "learning_rate": 7.549019607843137e-05, + "loss": 0.011568848043680191, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01164, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 0.5228955745697021, + "learning_rate": 7.647058823529411e-05, + "loss": 0.02045898698270321, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02067, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.5053377151489258, + "learning_rate": 7.745098039215687e-05, + "loss": 0.020432129502296448, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02064, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.35163524746894836, + "learning_rate": 7.843137254901961e-05, + "loss": 0.01403958909213543, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01414, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.6905986070632935, + "learning_rate": 7.941176470588235e-05, + "loss": 0.03329383581876755, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.03385, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 0.4459448456764221, + "learning_rate": 8.039215686274511e-05, + "loss": 0.006232030689716339, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00625, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.2638562321662903, + "learning_rate": 8.137254901960785e-05, + "loss": 0.006893642246723175, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00692, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 1.170509696006775, + "learning_rate": 8.23529411764706e-05, + "loss": 0.0398201048374176, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.04062, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.8102952837944031, + "learning_rate": 8.333333333333334e-05, + "loss": 0.018316198140382767, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01848, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.6812735795974731, + "learning_rate": 8.431372549019608e-05, + "loss": 0.02392837405204773, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02422, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.251705288887024, + "learning_rate": 8.529411764705883e-05, + "loss": 0.024196332320570946, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02449, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.5772960782051086, + "learning_rate": 8.627450980392158e-05, + "loss": 0.021560901775956154, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0218, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.6470917463302612, + "learning_rate": 8.725490196078432e-05, + "loss": 0.04491877555847168, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.04594, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.41440942883491516, + "learning_rate": 8.823529411764706e-05, + "loss": 0.0230612400919199, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02333, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 0.39720970392227173, + "learning_rate": 8.921568627450981e-05, + "loss": 0.013313441537320614, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0134, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.8898259401321411, + "learning_rate": 9.019607843137255e-05, + "loss": 0.015421045944094658, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01554, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 8.74992847442627, + "learning_rate": 9.11764705882353e-05, + "loss": 0.03271842002868652, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03326, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.18793171644210815, + "learning_rate": 9.215686274509804e-05, + "loss": 0.0015747153665870428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00158, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.9997888207435608, + "learning_rate": 9.313725490196079e-05, + "loss": 0.03334064409136772, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0339, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.6498851776123047, + "learning_rate": 9.411764705882353e-05, + "loss": 0.023218588903546333, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02349, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.5593264102935791, + "learning_rate": 9.509803921568627e-05, + "loss": 0.005530651658773422, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00555, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.8789476752281189, + "learning_rate": 9.607843137254903e-05, + "loss": 0.04554657265543938, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0466, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.5140370726585388, + "learning_rate": 9.705882352941177e-05, + "loss": 0.014101475477218628, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0142, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 0.5285096764564514, + "learning_rate": 9.80392156862745e-05, + "loss": 0.030611136928200722, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03108, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 0.6092981696128845, + "learning_rate": 9.901960784313727e-05, + "loss": 0.028998689725995064, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02942, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0235849618911743, + "learning_rate": 0.0001, + "loss": 0.025761116296052933, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0261, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.9383920431137085, + "learning_rate": 9.999994135965763e-05, + "loss": 0.02419213019311428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02449, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.8995917439460754, + "learning_rate": 9.999976543878332e-05, + "loss": 0.011181461624801159, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01124, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 3.9120519161224365, + "learning_rate": 9.999947223783559e-05, + "loss": 0.10826049745082855, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.11434, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 1.801079511642456, + "learning_rate": 9.999906175757857e-05, + "loss": 0.029263045638799667, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0297, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 3.040914535522461, + "learning_rate": 9.99985339990821e-05, + "loss": 0.04611361399292946, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04719, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.6034630537033081, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0116285290569067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0117, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.238577961921692, + "learning_rate": 9.999712665317819e-05, + "loss": 0.037302885204553604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03801, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.404407262802124, + "learning_rate": 9.999624706943868e-05, + "loss": 0.010177684016525745, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01023, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 1.1107022762298584, + "learning_rate": 9.999525021479544e-05, + "loss": 0.016156412661075592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01629, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 2.0084729194641113, + "learning_rate": 9.99941360918465e-05, + "loss": 0.022498758509755135, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02275, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.9237216711044312, + "learning_rate": 9.999290470349554e-05, + "loss": 0.009256366640329361, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0093, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 1.0012309551239014, + "learning_rate": 9.999155605295188e-05, + "loss": 0.01096038706600666, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01102, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 0.36062437295913696, + "learning_rate": 9.999009014373037e-05, + "loss": 0.0026956750079989433, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0027, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.09146630764007568, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0006923460750840604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00069, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 0.35573962330818176, + "learning_rate": 9.99868065648415e-05, + "loss": 0.0032197514083236456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00322, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.18824201822280884, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0015725638950243592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00157, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 2.2947869300842285, + "learning_rate": 9.998305400106009e-05, + "loss": 0.09606514871120453, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.10083, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.6851844787597656, + "learning_rate": 9.998100186186879e-05, + "loss": 0.006874916143715382, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0069, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.7458359003067017, + "learning_rate": 9.997883249150642e-05, + "loss": 0.020516997203230858, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02073, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 0.7431558966636658, + "learning_rate": 9.99765458956268e-05, + "loss": 0.014651929959654808, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01476, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.16357871890068054, + "learning_rate": 9.997414208018943e-05, + "loss": 0.003089388832449913, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00309, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.36700522899627686, + "learning_rate": 9.997162105145916e-05, + "loss": 0.01339892391115427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01349, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.22755062580108643, + "learning_rate": 9.996898281600642e-05, + "loss": 0.005444459617137909, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00546, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.339722216129303, + "learning_rate": 9.996622738070706e-05, + "loss": 0.004514522384852171, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00452, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 1.7278125286102295, + "learning_rate": 9.996335475274243e-05, + "loss": 0.024016786366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02431, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.47769469022750854, + "learning_rate": 9.996036493959926e-05, + "loss": 0.010395423509180546, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01045, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.4170791804790497, + "learning_rate": 9.99572579490697e-05, + "loss": 0.004119648598134518, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00413, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.37145885825157166, + "learning_rate": 9.995403378925132e-05, + "loss": 0.00920130219310522, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00924, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 0.06787586212158203, + "learning_rate": 9.995069246854705e-05, + "loss": 0.0012981402687728405, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.7351065874099731, + "learning_rate": 9.994723399566515e-05, + "loss": 0.021887682378292084, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02213, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.590771496295929, + "learning_rate": 9.994365837961926e-05, + "loss": 0.009705535136163235, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00975, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.3608993589878082, + "learning_rate": 9.993996562972823e-05, + "loss": 0.008060690015554428, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00809, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.48873215913772583, + "learning_rate": 9.993615575561626e-05, + "loss": 0.008283237926661968, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00832, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 0.7132001519203186, + "learning_rate": 9.99322287672128e-05, + "loss": 0.024729935452342033, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02504, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.09246033430099487, + "learning_rate": 9.992818467475254e-05, + "loss": 0.001992235891520977, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00199, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.7287611365318298, + "learning_rate": 9.992402348877528e-05, + "loss": 0.021320564672350883, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02155, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 0.2133086621761322, + "learning_rate": 9.991974522012612e-05, + "loss": 0.00337275885976851, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00338, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.37355223298072815, + "learning_rate": 9.99153498799552e-05, + "loss": 0.007823438383638859, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00785, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 0.06703885644674301, + "learning_rate": 9.991083747971783e-05, + "loss": 0.0005649570957757533, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00057, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.06440247595310211, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0006133198039606214, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00061, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 0.34707385301589966, + "learning_rate": 9.990146154639039e-05, + "loss": 0.0035533613990992308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00356, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 2.017024517059326, + "learning_rate": 9.98965980377362e-05, + "loss": 0.042595766484737396, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04352, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.785468339920044, + "learning_rate": 9.989161751788734e-05, + "loss": 0.002102769911289215, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0021, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 2.8202157020568848, + "learning_rate": 9.98865199998242e-05, + "loss": 0.01083361729979515, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01089, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 1.0493052005767822, + "learning_rate": 9.988130549683211e-05, + "loss": 0.014770902693271637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01488, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 2.157977819442749, + "learning_rate": 9.987597402250136e-05, + "loss": 0.009032780304551125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00907, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.029030824080109596, + "learning_rate": 9.987052559072696e-05, + "loss": 0.00013317995762918144, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 2.6754238605499268, + "learning_rate": 9.98649602157089e-05, + "loss": 0.033499784767627716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03407, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.9873471856117249, + "learning_rate": 9.985927791195179e-05, + "loss": 0.046224698424339294, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04731, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 1.0582185983657837, + "learning_rate": 9.985347869426513e-05, + "loss": 0.01951594091951847, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01971, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.6863466501235962, + "learning_rate": 9.984756257776302e-05, + "loss": 0.021039601415395737, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02126, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4057074785232544, + "learning_rate": 9.984152957786426e-05, + "loss": 0.008229494094848633, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00826, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.4019395411014557, + "learning_rate": 9.983537971029228e-05, + "loss": 0.010984211228787899, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01104, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.48016035556793213, + "learning_rate": 9.982911299107514e-05, + "loss": 0.012419218197464943, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.440166175365448, + "learning_rate": 9.982272943654535e-05, + "loss": 0.009114887565374374, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00916, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.20877902209758759, + "learning_rate": 9.981622906333999e-05, + "loss": 0.00270082987844944, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0027, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.6577605605125427, + "learning_rate": 9.980961188840055e-05, + "loss": 0.004442816600203514, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00445, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 0.13625678420066833, + "learning_rate": 9.9802877928973e-05, + "loss": 0.0019005294889211655, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0019, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.019028058275580406, + "learning_rate": 9.979602720260762e-05, + "loss": 0.0002803217794280499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00028, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.0662836953997612, + "learning_rate": 9.978905972715901e-05, + "loss": 0.0006815034430474043, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00068, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.9524247646331787, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00464291637763381, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00465, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 0.04438980668783188, + "learning_rate": 9.977477460195198e-05, + "loss": 0.00037421027082018554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 0.7781257629394531, + "learning_rate": 9.976745698942397e-05, + "loss": 0.0023766113445162773, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00238, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.4958593249320984, + "learning_rate": 9.976002270227351e-05, + "loss": 0.029030824080109596, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02946, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 2.34055495262146, + "learning_rate": 9.975247175987611e-05, + "loss": 0.04764559492468834, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0488, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.025128737092018127, + "learning_rate": 9.974480418191133e-05, + "loss": 0.0001649868063395843, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 2.489701986312866, + "learning_rate": 9.973701998836269e-05, + "loss": 0.17427006363868713, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.19038, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 1.9490054845809937, + "learning_rate": 9.972911919951763e-05, + "loss": 0.06718016415834427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06949, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 1.13898766040802, + "learning_rate": 9.972110183596752e-05, + "loss": 0.024798311293125153, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02511, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.4100293517112732, + "learning_rate": 9.971296791860748e-05, + "loss": 0.012426480650901794, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.5648359060287476, + "learning_rate": 9.970471746863644e-05, + "loss": 0.03443408012390137, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03503, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.3096520006656647, + "learning_rate": 9.969635050755704e-05, + "loss": 0.012037754990160465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.01211, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.42714670300483704, + "learning_rate": 9.968786705717555e-05, + "loss": 0.01219131425023079, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01227, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.5599139928817749, + "learning_rate": 9.967926713960188e-05, + "loss": 0.012130084447562695, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0122, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.6217107772827148, + "learning_rate": 9.967055077724944e-05, + "loss": 0.009348927065730095, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00939, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.0592292957007885, + "learning_rate": 9.966171799283513e-05, + "loss": 0.001284315949305892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00129, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 0.462398886680603, + "learning_rate": 9.965276880937933e-05, + "loss": 0.01244058832526207, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01252, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 8.241695404052734, + "learning_rate": 9.964370325020568e-05, + "loss": 0.05807769298553467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0598, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 1.0815411806106567, + "learning_rate": 9.96345213389412e-05, + "loss": 0.017220184206962585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01737, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 1.6311028003692627, + "learning_rate": 9.962522309951614e-05, + "loss": 0.005533762276172638, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00555, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 3.626565456390381, + "learning_rate": 9.961580855616392e-05, + "loss": 0.09052321314811707, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.09475, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 3.3724992275238037, + "learning_rate": 9.960627773342105e-05, + "loss": 0.00758603448048234, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00761, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 0.3157868981361389, + "learning_rate": 9.959663065612716e-05, + "loss": 0.003882938763126731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00389, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.03412550315260887, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0005760709755122662, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00058, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.52850741147995, + "learning_rate": 9.957698783875946e-05, + "loss": 0.010527489706873894, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01058, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.3949584364891052, + "learning_rate": 9.95669921498795e-05, + "loss": 0.007605592254549265, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00763, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 0.018618274480104446, + "learning_rate": 9.955688030883605e-05, + "loss": 0.0003067223879043013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00031, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.11938109248876572, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0008339773630723357, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00083, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 1.4441187381744385, + "learning_rate": 9.953630827597677e-05, + "loss": 0.015039840713143349, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01515, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.04352337867021561, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0008668563095852733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00087, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 10.194028854370117, + "learning_rate": 9.951527195464393e-05, + "loss": 0.01512847188860178, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01524, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.9922191500663757, + "learning_rate": 9.950457975414293e-05, + "loss": 0.012203281745314598, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01228, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 1.3960785865783691, + "learning_rate": 9.949377156413995e-05, + "loss": 0.010049878619611263, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0101, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.6242722272872925, + "learning_rate": 9.948284741280371e-05, + "loss": 0.020798590034246445, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02102, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.9250545501708984, + "learning_rate": 9.947180732860518e-05, + "loss": 0.02078751102089882, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02101, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.44395211338996887, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0027410993352532387, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00274, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.06538265198469162, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0013557756319642067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00136, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.8934104442596436, + "learning_rate": 9.943799176807672e-05, + "loss": 0.009296886622905731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00934, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.285216748714447, + "learning_rate": 9.942648824318006e-05, + "loss": 0.0030592025723308325, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00306, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.6151894330978394, + "learning_rate": 9.941486893230645e-05, + "loss": 0.00584901450201869, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00587, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.3476015329360962, + "learning_rate": 9.940313386573859e-05, + "loss": 0.005612781271338463, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00563, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.04463421553373337, + "learning_rate": 9.939128307406084e-05, + "loss": 0.0004844452196266502, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00048, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.14716975390911102, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0013532615266740322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00135, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.013975728303194046, + "learning_rate": 9.936723443922109e-05, + "loss": 0.00017458554066251963, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.23583655059337616, + "learning_rate": 9.935503665873555e-05, + "loss": 0.0013064221711829305, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00131, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 0.19946330785751343, + "learning_rate": 9.934272327849287e-05, + "loss": 0.0016659648390486836, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00167, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 2.5991828441619873, + "learning_rate": 9.933029433058463e-05, + "loss": 0.03332412242889404, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03389, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.022260449826717377, + "learning_rate": 9.931774984740362e-05, + "loss": 0.00019467048696242273, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00019, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 1.043959140777588, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0496981106698513, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05095, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.009332562796771526, + "learning_rate": 9.929231440629993e-05, + "loss": 0.0001336124842055142, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00013, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.028200708329677582, + "learning_rate": 9.927942351466806e-05, + "loss": 0.00035777001176029444, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00036, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.06076425686478615, + "learning_rate": 9.926641722034484e-05, + "loss": 0.001383051392622292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00138, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.12561249732971191, + "learning_rate": 9.925329555722777e-05, + "loss": 0.0026724478229880333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00268, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 0.5528528094291687, + "learning_rate": 9.924005855951503e-05, + "loss": 0.01256745308637619, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01265, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 2.376209259033203, + "learning_rate": 9.922670626170537e-05, + "loss": 0.017799941822886467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01796, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.2762843370437622, + "learning_rate": 9.921323869859805e-05, + "loss": 0.007115472108125687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00714, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.24070966243743896, + "learning_rate": 9.919965590529272e-05, + "loss": 0.0015583797357976437, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00156, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.10395435988903046, + "learning_rate": 9.918595791718938e-05, + "loss": 0.0007914248271845281, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00079, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.5730319023132324, + "learning_rate": 9.917214476998825e-05, + "loss": 0.005572602618485689, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00559, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.1355881541967392, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0004524950636550784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00045, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.07415846735239029, + "learning_rate": 9.914417314259398e-05, + "loss": 0.0004993464681319892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0005, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 0.22872294485569, + "learning_rate": 9.913001473530153e-05, + "loss": 0.0009260782971978188, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00093, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.006719402503222227, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001365313510177657, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.020894810557365417, + "learning_rate": 9.910135291802682e-05, + "loss": 0.00035784178180620074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00036, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.06920765340328217, + "learning_rate": 9.908684958274405e-05, + "loss": 0.000399862794438377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0004, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.008416767232120037, + "learning_rate": 9.907223134666332e-05, + "loss": 4.7169265599222854e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00005, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 0.013850455172359943, + "learning_rate": 9.905749824788326e-05, + "loss": 0.00019475985027384013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00019, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.0023701407480984926, + "learning_rate": 9.904265032480177e-05, + "loss": 3.626237958087586e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.0015644921222701669, + "learning_rate": 9.902768761611609e-05, + "loss": 3.6560850276146084e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 0.54908686876297, + "learning_rate": 9.901261016082262e-05, + "loss": 0.02532411366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02565, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.00214971462264657, + "learning_rate": 9.899741799821673e-05, + "loss": 3.0114068067632616e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0071777780540287495, + "learning_rate": 9.898211116789284e-05, + "loss": 5.1465347496559843e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 1.685914158821106, + "learning_rate": 9.896668970974419e-05, + "loss": 0.014849710278213024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01496, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 1.67842698097229, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0018469173228368163, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.005261159967631102, + "learning_rate": 9.893550307103913e-05, + "loss": 8.17667314549908e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.023177370429039, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00030886579770594835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00031, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.3134678304195404, + "learning_rate": 9.890385840722048e-05, + "loss": 0.0008985826279968023, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0009, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.7450703382492065, + "learning_rate": 9.888786441879897e-05, + "loss": 0.017146039754152298, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01729, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 0.008559435606002808, + "learning_rate": 9.887175604818206e-05, + "loss": 0.0001242949510924518, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.00012, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 1.7312885522842407, + "learning_rate": 9.885553333735204e-05, + "loss": 0.008018076419830322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00805, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 1.6031489372253418, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0027930131182074547, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0028, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.7863869071006775, + "learning_rate": 9.882274506447146e-05, + "loss": 0.02333652786910534, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02361, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.6194532513618469, + "learning_rate": 9.88061795878749e-05, + "loss": 0.003525766544044018, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00353, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 1.360031008720398, + "learning_rate": 9.878949994197307e-05, + "loss": 0.017261315137147903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01741, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03367060795426369, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0006716687930747867, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00067, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.1465068757534027, + "learning_rate": 9.875579831643538e-05, + "loss": 0.0024646068923175335, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00247, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.6451879739761353, + "learning_rate": 9.873877642463397e-05, + "loss": 0.016013173386454582, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01614, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.5427649617195129, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010463962331414223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01052, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.14428693056106567, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0027021225541830063, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00271, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.41795286536216736, + "learning_rate": 9.868702696634756e-05, + "loss": 0.014849536120891571, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01496, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.33399561047554016, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0074952589347958565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00752, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.2544633150100708, + "learning_rate": 9.865195795873484e-05, + "loss": 0.005140242632478476, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00515, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.6971493363380432, + "learning_rate": 9.863425278095409e-05, + "loss": 0.02200455032289028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02225, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.15962868928909302, + "learning_rate": 9.861643388195015e-05, + "loss": 0.002751844236627221, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00276, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.3089817464351654, + "learning_rate": 9.859850130816326e-05, + "loss": 0.0019627867732197046, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00196, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.03729058429598808, + "learning_rate": 9.858045510632996e-05, + "loss": 0.0004204344586469233, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00042, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.2555536925792694, + "learning_rate": 9.856229532348296e-05, + "loss": 0.002585819223895669, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00259, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.017389044165611267, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00010104697139468044, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 0.002870214870199561, + "learning_rate": 9.852563520435858e-05, + "loss": 5.9336227423045784e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.2427501380443573, + "learning_rate": 9.850713496362614e-05, + "loss": 0.004451706539839506, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00446, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.005748335737735033, + "learning_rate": 9.848852133296972e-05, + "loss": 8.568367775296792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0012240263167768717, + "learning_rate": 9.846979436090085e-05, + "loss": 4.23779092670884e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.01661101169884205, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0001144747220678255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.020272312685847282, + "learning_rate": 9.843200058804868e-05, + "loss": 0.00017771853890735656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00018, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.033239174634218216, + "learning_rate": 9.841293388576495e-05, + "loss": 0.00044481243821792305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00044, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5434594750404358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.009829508140683174, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00988, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.3308855891227722, + "learning_rate": 9.837446109794382e-05, + "loss": 0.004846261814236641, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00486, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.003991277422755957, + "learning_rate": 9.835505511267567e-05, + "loss": 8.371799776796252e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.10116580128669739, + "learning_rate": 9.83355361338397e-05, + "loss": 0.0005483412533067167, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00055, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.003369405400007963, + "learning_rate": 9.831590421230704e-05, + "loss": 7.847918459447101e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00008, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.011247020214796066, + "learning_rate": 9.829615939924308e-05, + "loss": 0.000140060976264067, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.009074223227798939, + "learning_rate": 9.827630174610754e-05, + "loss": 9.554257121635601e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0001, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.000541043933480978, + "learning_rate": 9.825633130465418e-05, + "loss": 1.697065818007104e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.014442048035562038, + "learning_rate": 9.823624812693069e-05, + "loss": 9.099359886022285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00009, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.013665883801877499, + "learning_rate": 9.821605226527862e-05, + "loss": 0.00011380278010619804, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00011, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.0006537620793096721, + "learning_rate": 9.819574377233317e-05, + "loss": 1.6407084331149235e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.01291027944535017, + "learning_rate": 9.817532270102309e-05, + "loss": 6.133162969490513e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00006, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.5513811111450195, + "learning_rate": 9.815478910457054e-05, + "loss": 0.007841670885682106, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00787, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.0005016659270040691, + "learning_rate": 9.813414303649097e-05, + "loss": 1.6098865671665408e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 0.1864936798810959, + "learning_rate": 9.811338455059291e-05, + "loss": 0.0015166733646765351, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00152, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.0026293513365089893, + "learning_rate": 9.809251370097791e-05, + "loss": 5.2753948693862185e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00005, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.41635486483573914, + "learning_rate": 9.807153054204036e-05, + "loss": 0.02190607599914074, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02215, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 1.0917211771011353, + "learning_rate": 9.805043512846735e-05, + "loss": 0.019963297992944717, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02016, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.06857825815677643, + "learning_rate": 9.802922751523855e-05, + "loss": 0.00030228254036046565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.031100548803806305, + "learning_rate": 9.800790775762605e-05, + "loss": 0.0004046057292725891, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0004, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.47315219044685364, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0014745643129572272, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00148, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.0773354172706604, + "learning_rate": 9.796493203179937e-05, + "loss": 0.0008912066696211696, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00089, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.1289902925491333, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0018994644051417708, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0019, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.054183945059776306, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0005117067485116422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00051, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.12445089966058731, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0022513207513839006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00225, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 9.98, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.20701012015342712, + "learning_rate": 9.787763731193733e-05, + "loss": 0.003331368323415518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00334, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.6194630265235901, + "learning_rate": 9.785553411578963e-05, + "loss": 0.03162220120429993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03213, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.2134065181016922, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0037704987917095423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00378, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.040715303272008896, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0002603174070827663, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00026, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 0.7002236247062683, + "learning_rate": 9.778855460904167e-05, + "loss": 0.006636213511228561, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00666, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.007663452997803688, + "learning_rate": 9.77660049946513e-05, + "loss": 0.00024576610303483903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00025, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.023272940889000893, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0005693582352250814, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00057, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.026790525764226913, + "learning_rate": 9.77205714498407e-05, + "loss": 0.0007213435601443052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.004755170550197363, + "learning_rate": 9.769768763783107e-05, + "loss": 0.00017550830671098083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00018, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 0.016630757600069046, + "learning_rate": 9.767469254550936e-05, + "loss": 0.0002396691415924579, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00024, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 0.09570515155792236, + "learning_rate": 9.765158623280625e-05, + "loss": 0.0009529241942800581, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00095, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.5903910994529724, + "learning_rate": 9.762836875994225e-05, + "loss": 0.005048507358878851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00506, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.008239318616688251, + "learning_rate": 9.760504018742764e-05, + "loss": 0.00023036239144857973, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00023, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.060113124549388885, + "learning_rate": 9.75816005760622e-05, + "loss": 0.0003171982243657112, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00032, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.008316988125443459, + "learning_rate": 9.755804998693513e-05, + "loss": 0.00014665098569821566, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00015, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.21268479526042938, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0025118833873420954, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00252, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.005383700132369995, + "learning_rate": 9.751061612119887e-05, + "loss": 0.00013681002019438893, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00014, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.4226222336292267, + "learning_rate": 9.748673296821358e-05, + "loss": 0.00028868031222373247, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00029, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.007631849963217974, + "learning_rate": 9.746273908471418e-05, + "loss": 0.000154903216753155, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.09700846672058105, + "learning_rate": 9.743863453323441e-05, + "loss": 0.0013645484577864408, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.018550626933574677, + "learning_rate": 9.741441937659645e-05, + "loss": 0.0002988259366247803, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0003, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.005299921613186598, + "learning_rate": 9.73900936779108e-05, + "loss": 8.563048322685063e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.015369605273008347, + "learning_rate": 9.736565750057593e-05, + "loss": 0.00014232251851353794, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.006885336712002754, + "learning_rate": 9.734111090827836e-05, + "loss": 0.00014437930076383054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00014, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.3322601020336151, + "learning_rate": 9.731645396499232e-05, + "loss": 0.0058242324739694595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00584, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.008417945355176926, + "learning_rate": 9.729168673497969e-05, + "loss": 9.954217239283025e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0001, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.3274463713169098, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0031569483689963818, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00316, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.011693589389324188, + "learning_rate": 9.724182167325899e-05, + "loss": 7.778844883432612e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00008, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 1.3683384656906128, + "learning_rate": 9.721672397151109e-05, + "loss": 0.010511158034205437, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01057, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.005915132351219654, + "learning_rate": 9.719151624295665e-05, + "loss": 6.681385275442153e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00007, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.00782559160143137, + "learning_rate": 9.716619855329296e-05, + "loss": 8.550586790079251e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00009, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.0015690920408815145, + "learning_rate": 9.714077096850394e-05, + "loss": 2.1982301404932514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.0027190272230654955, + "learning_rate": 9.711523355485992e-05, + "loss": 2.6376750611234456e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 0.21495184302330017, + "learning_rate": 9.708958637891746e-05, + "loss": 0.0006077390280552208, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 0.0004555387713480741, + "learning_rate": 9.706382950751918e-05, + "loss": 1.7097307136282325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 0.02135647088289261, + "learning_rate": 9.703796300779361e-05, + "loss": 0.00014291358820628375, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 0.011736172251403332, + "learning_rate": 9.7011986947155e-05, + "loss": 3.78161967091728e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00004, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.874617338180542, + "learning_rate": 9.698590139330309e-05, + "loss": 0.006245121825486422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00626, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 0.027552802115678787, + "learning_rate": 9.695970641422309e-05, + "loss": 0.00019723488367162645, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 0.00035697652492672205, + "learning_rate": 9.693340207818528e-05, + "loss": 1.3179367670090869e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 0.08428651839494705, + "learning_rate": 9.690698845374505e-05, + "loss": 0.0008772820001468062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00088, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 0.00604958226904273, + "learning_rate": 9.688046560974256e-05, + "loss": 8.847641583997756e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 0.00039003140409477055, + "learning_rate": 9.685383361530262e-05, + "loss": 1.5304913176805712e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 0.00023623408924322575, + "learning_rate": 9.682709253983455e-05, + "loss": 8.844612239045091e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.00048069472541101277, + "learning_rate": 9.680024245303193e-05, + "loss": 1.667063588683959e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.16455329954624176, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0005412459140643477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00054, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.012845818884670734, + "learning_rate": 9.674621552561782e-05, + "loss": 0.00013661464618053287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.7110337615013123, + "learning_rate": 9.671903882581331e-05, + "loss": 0.005711458623409271, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00573, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.007893380708992481, + "learning_rate": 9.669175339628791e-05, + "loss": 9.75146540440619e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.10780492424964905, + "learning_rate": 9.66643593081539e-05, + "loss": 0.0005473099881783128, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00055, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.0010501379147171974, + "learning_rate": 9.663685663280679e-05, + "loss": 2.7556767236092128e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.288404256105423, + "learning_rate": 9.66092454419251e-05, + "loss": 0.0018175432924181223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00182, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 1.9734091758728027, + "learning_rate": 9.65815258074701e-05, + "loss": 0.03248640149831772, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03302, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 0.2764054536819458, + "learning_rate": 9.655369780168579e-05, + "loss": 0.0009657267364673316, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00097, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.0005600836011581123, + "learning_rate": 9.652576149709852e-05, + "loss": 1.6435071302112192e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 0.0008129188208840787, + "learning_rate": 9.649771696651697e-05, + "loss": 2.3179851268650964e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.0014548521721735597, + "learning_rate": 9.646956428303183e-05, + "loss": 1.6358961147489026e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.5601457953453064, + "learning_rate": 9.644130352001567e-05, + "loss": 0.003099913941696286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0031, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.00297771324403584, + "learning_rate": 9.641293475112275e-05, + "loss": 4.133052425459027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.0016736306715756655, + "learning_rate": 9.638445805028884e-05, + "loss": 4.893036748399027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.001026933896355331, + "learning_rate": 9.635587349173095e-05, + "loss": 2.9338965759961866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.008905365131795406, + "learning_rate": 9.632718114994726e-05, + "loss": 9.410006168764085e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.44935929775238037, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0031317835673689842, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00314, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.004555090330541134, + "learning_rate": 9.626947341609934e-05, + "loss": 0.00011855485354317352, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00012, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.001394708058796823, + "learning_rate": 9.624045817443519e-05, + "loss": 4.678318146034144e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.0037189542781561613, + "learning_rate": 9.621133545034491e-05, + "loss": 0.00011213941616006196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.1473202258348465, + "learning_rate": 9.61821053197293e-05, + "loss": 0.0005101395072415471, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00051, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.012220866046845913, + "learning_rate": 9.615276785876898e-05, + "loss": 0.0002749463019426912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00027, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.007556390482932329, + "learning_rate": 9.612332314392435e-05, + "loss": 0.00013154096086509526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.002842485439032316, + "learning_rate": 9.609377125193535e-05, + "loss": 7.768635987304151e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00008, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.04467956721782684, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00022737993276678026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 1.0502334833145142, + "learning_rate": 9.603434624488023e-05, + "loss": 0.015947995707392693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01608, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.014036186039447784, + "learning_rate": 9.600447328468982e-05, + "loss": 0.00013877129822503775, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.6235236525535583, + "learning_rate": 9.597449345710597e-05, + "loss": 0.015155073255300522, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01527, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.0999632477760315, + "learning_rate": 9.594440684026324e-05, + "loss": 0.0012880185386165977, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00129, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.02786194533109665, + "learning_rate": 9.591421351257449e-05, + "loss": 0.0003391870704945177, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00034, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.5290789008140564, + "learning_rate": 9.588391355273066e-05, + "loss": 0.007406376302242279, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00743, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.01936807669699192, + "learning_rate": 9.585350703970069e-05, + "loss": 0.00025529557024128735, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00026, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 0.005768527276813984, + "learning_rate": 9.582299405273118e-05, + "loss": 8.689028618391603e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.9624814391136169, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00811285525560379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00815, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.018384141847491264, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0003791853378061205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00038, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.6165330410003662, + "learning_rate": 9.573081704481236e-05, + "loss": 0.013609700836241245, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0137, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.0840073898434639, + "learning_rate": 9.569987896009734e-05, + "loss": 0.00045918801333755255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00046, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.008723951876163483, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00011756666208384559, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00012, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.012961777858436108, + "learning_rate": 9.563768465093079e-05, + "loss": 0.00030332955066114664, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0003, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.02426735870540142, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00048618094297125936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00049, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.1119721531867981, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0008597768610343337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00086, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.03705863282084465, + "learning_rate": 9.554359905560886e-05, + "loss": 0.0003950979153160006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0004, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.6851837038993835, + "learning_rate": 9.551202574875235e-05, + "loss": 0.011227660812437534, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01129, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.06478452682495117, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0012987275840714574, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.886103630065918, + "learning_rate": 9.544856246572735e-05, + "loss": 0.019783541560173035, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01998, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.03494488075375557, + "learning_rate": 9.541667265495922e-05, + "loss": 0.0006417686236090958, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.1211373507976532, + "learning_rate": 9.538467750874569e-05, + "loss": 0.0016129965661093593, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00161, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.9197566509246826, + "learning_rate": 9.535257711047364e-05, + "loss": 0.01892932318150997, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01911, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.3712734878063202, + "learning_rate": 9.532037154380439e-05, + "loss": 0.004730743821710348, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00474, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.13118499517440796, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0026065458077937365, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00261, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.41263192892074585, + "learning_rate": 9.525564524128947e-05, + "loss": 0.02503504976630211, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02535, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.05777617543935776, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0012999713653698564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0013, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.16045935451984406, + "learning_rate": 9.519049927596896e-05, + "loss": 0.006284795701503754, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0063, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.10557752847671509, + "learning_rate": 9.515776913181807e-05, + "loss": 0.0018496124539524317, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00185, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.1812046468257904, + "learning_rate": 9.512493432698582e-05, + "loss": 0.003738866187632084, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00375, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.11412262171506882, + "learning_rate": 9.509199494704748e-05, + "loss": 0.00221292139030993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00222, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.05267500877380371, + "learning_rate": 9.505895107785091e-05, + "loss": 0.0010225172154605389, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00102, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.23904849588871002, + "learning_rate": 9.502580280551627e-05, + "loss": 0.005414942279458046, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00543, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.013456333428621292, + "learning_rate": 9.499255021643582e-05, + "loss": 0.00013312777446117252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00013, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.0178237147629261, + "learning_rate": 9.495919339727372e-05, + "loss": 0.0003349714388605207, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00034, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.049912769347429276, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0004279202257748693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.014288226142525673, + "learning_rate": 9.489216741671916e-05, + "loss": 0.00036869742325507104, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00037, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 0.00872453860938549, + "learning_rate": 9.485849843001232e-05, + "loss": 6.138686876511201e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.0013658624375239015, + "learning_rate": 9.48247255625946e-05, + "loss": 3.532581831677817e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.0017636138945817947, + "learning_rate": 9.479084890248611e-05, + "loss": 2.53205780609278e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.000683065562043339, + "learning_rate": 9.475686853797749e-05, + "loss": 1.986018833122216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.016209598630666733, + "learning_rate": 9.472278455762962e-05, + "loss": 0.00014525593724101782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00015, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.020629821345210075, + "learning_rate": 9.468859705027345e-05, + "loss": 8.353606972377747e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.0009277141070924699, + "learning_rate": 9.465430610500977e-05, + "loss": 1.4405310139409266e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.0002464439603500068, + "learning_rate": 9.461991181120889e-05, + "loss": 9.015106115839444e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.000667985004838556, + "learning_rate": 9.458541425851051e-05, + "loss": 2.003746703849174e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.002300550462678075, + "learning_rate": 9.455081353682347e-05, + "loss": 4.4835043809143826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00004, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.0017511912155896425, + "learning_rate": 9.451610973632543e-05, + "loss": 6.716565621900372e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.0028233909979462624, + "learning_rate": 9.448130294746276e-05, + "loss": 2.4473858502460644e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.00019360017904546112, + "learning_rate": 9.444639326095022e-05, + "loss": 5.723140020563733e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 0.00023336702724918723, + "learning_rate": 9.441138076777071e-05, + "loss": 4.722754511021776e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.0003432357043493539, + "learning_rate": 9.437626555917512e-05, + "loss": 9.122491974267177e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.23666708171367645, + "learning_rate": 9.434104772668203e-05, + "loss": 0.0011873405892401934, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00119, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.0001242521102540195, + "learning_rate": 9.430572736207746e-05, + "loss": 2.8345539249130525e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.00043305609142407775, + "learning_rate": 9.427030455741466e-05, + "loss": 1.0893121725530364e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005291244946420193, + "learning_rate": 9.423477940501391e-05, + "loss": 3.527114313328639e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00004, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0002391930902376771, + "learning_rate": 9.419915199746214e-05, + "loss": 4.351905317889759e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.0001795155694708228, + "learning_rate": 9.416342242761287e-05, + "loss": 4.172787157585844e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.00027239706832915545, + "learning_rate": 9.412759078858583e-05, + "loss": 6.559062967426144e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.004924969747662544, + "learning_rate": 9.409165717376678e-05, + "loss": 1.9311535652377643e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00002, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00028949262923561037, + "learning_rate": 9.405562167680723e-05, + "loss": 5.919015166000463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.0019513516454026103, + "learning_rate": 9.401948439162429e-05, + "loss": 6.4454043240402825e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.00024267581466119736, + "learning_rate": 9.398324541240028e-05, + "loss": 2.6868240183830494e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.0005200989544391632, + "learning_rate": 9.394690483358256e-05, + "loss": 5.118726221553516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.00012520125892478973, + "learning_rate": 9.391046274988333e-05, + "loss": 2.580744194347062e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.00037211537710390985, + "learning_rate": 9.387391925627933e-05, + "loss": 6.75070896249963e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.17, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.006821396294981241, + "learning_rate": 9.383727444801156e-05, + "loss": 1.0072410987049807e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.005923285149037838, + "learning_rate": 9.380052842058508e-05, + "loss": 1.7001904780045152e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0007364701014012098, + "learning_rate": 9.376368126976879e-05, + "loss": 6.04315209784545e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.0021281663794070482, + "learning_rate": 9.37267330915951e-05, + "loss": 4.2355783079983667e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.00013200279499869794, + "learning_rate": 9.368968398235973e-05, + "loss": 2.4993578335852362e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.62, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 9.607620449969545e-05, + "learning_rate": 9.365253403862147e-05, + "loss": 2.0314462290116353e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.00013611785834655166, + "learning_rate": 9.361528335720188e-05, + "loss": 2.2868246105645085e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0671246275305748, + "learning_rate": 9.35779320351851e-05, + "loss": 5.1696148148039356e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.00018044775060843676, + "learning_rate": 9.354048016991751e-05, + "loss": 3.048619419132592e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.00013236024824436754, + "learning_rate": 9.35029278590076e-05, + "loss": 3.4778872759488877e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.0, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.004715391434729099, + "learning_rate": 9.346527520032561e-05, + "loss": 2.927492005255772e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.00028085883241146803, + "learning_rate": 9.342752229200325e-05, + "loss": 4.623073891707463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 8.980643178801984e-05, + "learning_rate": 9.338966923243363e-05, + "loss": 1.8482477344150539e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.003043609904125333, + "learning_rate": 9.335171612027078e-05, + "loss": 1.538229662401136e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 8.966683526523411e-05, + "learning_rate": 9.331366305442954e-05, + "loss": 2.0870693333563395e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.0009604596416465938, + "learning_rate": 9.32755101340852e-05, + "loss": 3.408649718039669e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.00011633319081738591, + "learning_rate": 9.323725745867336e-05, + "loss": 3.0257524485932663e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.0010066182585433125, + "learning_rate": 9.319890512788957e-05, + "loss": 8.530637387593742e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.00016301196592394263, + "learning_rate": 9.316045324168912e-05, + "loss": 2.349399892409565e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.003284434787929058, + "learning_rate": 9.312190190028672e-05, + "loss": 2.500721711840015e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.00014885724522173405, + "learning_rate": 9.308325120415636e-05, + "loss": 3.3532028282934334e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 9.280376980314031e-05, + "learning_rate": 9.30445012540309e-05, + "loss": 1.334464627689158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 8.563206938561052e-05, + "learning_rate": 9.300565215090192e-05, + "loss": 1.549241460452322e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.00010646737791830674, + "learning_rate": 9.296670399601942e-05, + "loss": 1.6117182894959114e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 9.888842760119587e-05, + "learning_rate": 9.292765689089152e-05, + "loss": 1.9165411231369944e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 7.287397602340207e-05, + "learning_rate": 9.288851093728427e-05, + "loss": 2.075230213449686e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.00041501791565679014, + "learning_rate": 9.284926623722131e-05, + "loss": 3.292028623036458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 5.636964851873927e-05, + "learning_rate": 9.280992289298363e-05, + "loss": 1.1177535270689987e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.0004036540340166539, + "learning_rate": 9.277048100710934e-05, + "loss": 4.003003596153576e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 5.150034121470526e-05, + "learning_rate": 9.273094068239336e-05, + "loss": 1.1050038892790326e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.0008515650406479836, + "learning_rate": 9.269130202188718e-05, + "loss": 9.22057733987458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.818722128868103, + "learning_rate": 9.265156512889854e-05, + "loss": 0.0029519246891140938, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00296, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 7.819741585990414e-05, + "learning_rate": 9.261173010699122e-05, + "loss": 2.333828206246835e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.0005888771847821772, + "learning_rate": 9.257179705998475e-05, + "loss": 1.9795390926447e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 4.292320954846218e-05, + "learning_rate": 9.253176609195413e-05, + "loss": 1.0174778708460508e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 4.8053876525955275e-05, + "learning_rate": 9.249163730722958e-05, + "loss": 1.1561764949874487e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.00020360939379315823, + "learning_rate": 9.245141081039623e-05, + "loss": 2.147836312360596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 5.6159336963901296e-05, + "learning_rate": 9.241108670629387e-05, + "loss": 1.3456724445859436e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.4064067304134369, + "learning_rate": 9.23706651000167e-05, + "loss": 0.0016359895234927535, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00164, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.00012425552995409817, + "learning_rate": 9.233014609691304e-05, + "loss": 1.811459014788852e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.001742662163451314, + "learning_rate": 9.2289529802585e-05, + "loss": 2.9671325592062203e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.0006759302341379225, + "learning_rate": 9.224881632288832e-05, + "loss": 5.512976258614799e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.00019010427058674395, + "learning_rate": 9.2208005763932e-05, + "loss": 1.384292886541516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.002026048256084323, + "learning_rate": 9.2167098232078e-05, + "loss": 1.1068831554439384e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 4.712811278295703e-05, + "learning_rate": 9.21260938339411e-05, + "loss": 8.564664426558011e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.000333633302943781, + "learning_rate": 9.208499267638847e-05, + "loss": 4.601421096595004e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 4.1058650822378695e-05, + "learning_rate": 9.204379486653952e-05, + "loss": 7.030776600913669e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 4.685380918090232e-05, + "learning_rate": 9.20025005117655e-05, + "loss": 8.994193194666877e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 4.656754754250869e-05, + "learning_rate": 9.19611097196893e-05, + "loss": 1.0813282642629929e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 1.0787988901138306, + "learning_rate": 9.191962259818518e-05, + "loss": 0.024499906226992607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0248, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.00020449870498850942, + "learning_rate": 9.18780392553784e-05, + "loss": 2.7060591492045205e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.0011033477494493127, + "learning_rate": 9.183635979964503e-05, + "loss": 9.57420161284972e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.002774628810584545, + "learning_rate": 9.179458433961166e-05, + "loss": 3.007879786309786e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.026837196201086044, + "learning_rate": 9.1752712984155e-05, + "loss": 0.0002948661567643285, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00029, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.01849069632589817, + "learning_rate": 9.17107458424018e-05, + "loss": 0.00011084417928941548, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.009761124849319458, + "learning_rate": 9.166868302372835e-05, + "loss": 3.652252780739218e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.08665445446968079, + "learning_rate": 9.162652463776038e-05, + "loss": 0.0008891914039850235, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.03040151111781597, + "learning_rate": 9.158427079437264e-05, + "loss": 0.00030500703724101186, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.05672994256019592, + "learning_rate": 9.154192160368866e-05, + "loss": 0.0007530992152169347, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.003426315262913704, + "learning_rate": 9.149947717608052e-05, + "loss": 1.5134808563743718e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.004125513136386871, + "learning_rate": 9.145693762216845e-05, + "loss": 6.913553079357371e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00007, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.04262302443385124, + "learning_rate": 9.141430305282065e-05, + "loss": 0.000580799242015928, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00058, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 0.013481490314006805, + "learning_rate": 9.137157357915291e-05, + "loss": 8.695045835338533e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00009, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 4.995663166046143, + "learning_rate": 9.132874931252841e-05, + "loss": 0.02319028601050377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02346, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.0015562125481665134, + "learning_rate": 9.128583036455732e-05, + "loss": 2.1520958398468792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.0061469124630093575, + "learning_rate": 9.124281684709663e-05, + "loss": 8.727848035050556e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00009, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.003409374738112092, + "learning_rate": 9.119970887224978e-05, + "loss": 5.775516910944134e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00006, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.015050097368657589, + "learning_rate": 9.11565065523664e-05, + "loss": 0.0001365312928101048, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.01874382048845291, + "learning_rate": 9.111321000004195e-05, + "loss": 0.00020053039770573378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.002161014126613736, + "learning_rate": 9.106981932811758e-05, + "loss": 3.2826297683641315e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.0013815263519063592, + "learning_rate": 9.102633464967965e-05, + "loss": 3.158616891596466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.13355329632759094, + "learning_rate": 9.098275607805957e-05, + "loss": 0.00038878218038007617, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00039, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.04, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 0.0038680124562233686, + "learning_rate": 9.093908372683343e-05, + "loss": 7.378710870398208e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.3629514276981354, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0006530214450322092, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0069854846224188805, + "learning_rate": 9.085145814108921e-05, + "loss": 3.534342613420449e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0013717467663809657, + "learning_rate": 9.08075051349442e-05, + "loss": 2.084069637930952e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.010476332157850266, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0001456337922718376, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00015, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.002277064835652709, + "learning_rate": 9.071931926886792e-05, + "loss": 2.8683749405900016e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00003, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.4141025245189667, + "learning_rate": 9.067508663876997e-05, + "loss": 0.0020133755169808865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00202, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.001090523204766214, + "learning_rate": 9.063076103092557e-05, + "loss": 2.1892830773140304e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 0.0011893577175214887, + "learning_rate": 9.05863425608578e-05, + "loss": 1.7626425687922165e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.3477942645549774, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002018592320382595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00202, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 165520 + }, + { + "epoch": 8.015625, + "grad_norm": 0.0017783264629542828, + "learning_rate": 9.049722749735415e-05, + "loss": 1.2956855243828613e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 513, + "tokens/total": 7745176, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 165842 + }, + { + "epoch": 8.03125, + "grad_norm": 0.0009507329086773098, + "learning_rate": 9.04525311361733e-05, + "loss": 1.1546111636562273e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 514, + "tokens/total": 7760316, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 166164 + }, + { + "epoch": 8.046875, + "grad_norm": 0.013517077080905437, + "learning_rate": 9.04077423772785e-05, + "loss": 6.299532833509147e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 515, + "tokens/total": 7775516, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 166490 + }, + { + "epoch": 8.0625, + "grad_norm": 0.0008650442468933761, + "learning_rate": 9.036286133739987e-05, + "loss": 2.090480484184809e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 516, + "tokens/total": 7790668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 166814 + }, + { + "epoch": 8.078125, + "grad_norm": 0.03777744248509407, + "learning_rate": 9.031788813350807e-05, + "loss": 0.00017636040865909308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00018, + "step": 517, + "tokens/total": 7805712, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 167137 + }, + { + "epoch": 8.09375, + "grad_norm": 0.06616051495075226, + "learning_rate": 9.027282288281396e-05, + "loss": 0.00020225218031555414, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 518, + "tokens/total": 7820848, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 167461 + }, + { + "epoch": 8.109375, + "grad_norm": 0.0019223307026550174, + "learning_rate": 9.022766570276829e-05, + "loss": 1.723412970022764e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 519, + "tokens/total": 7835976, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 167784 + }, + { + "epoch": 8.125, + "grad_norm": 0.0008735855808481574, + "learning_rate": 9.018241671106134e-05, + "loss": 1.880908530438319e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 520, + "tokens/total": 7851168, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 168106 + }, + { + "epoch": 8.140625, + "grad_norm": 0.37315821647644043, + "learning_rate": 9.013707602562279e-05, + "loss": 0.0009505169582553208, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00095, + "step": 521, + "tokens/total": 7866244, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 168434 + }, + { + "epoch": 8.15625, + "grad_norm": 0.00021756299247499555, + "learning_rate": 9.009164376462119e-05, + "loss": 7.845479558454826e-06, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 522, + "tokens/total": 7881132, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 168757 + }, + { + "epoch": 8.171875, + "grad_norm": 0.002056313678622246, + "learning_rate": 9.004612004646383e-05, + "loss": 2.414161281194538e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 523, + "tokens/total": 7896264, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 169077 + }, + { + "epoch": 8.1875, + "grad_norm": 0.06747874617576599, + "learning_rate": 9.00005049897963e-05, + "loss": 0.00028232645126990974, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00028, + "step": 524, + "tokens/total": 7911500, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 169403 + }, + { + "epoch": 8.203125, + "grad_norm": 0.0003100395551882684, + "learning_rate": 8.995479871350229e-05, + "loss": 1.054319909599144e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 525, + "tokens/total": 7926620, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 169726 + }, + { + "epoch": 8.21875, + "grad_norm": 0.008461945690214634, + "learning_rate": 8.990900133670319e-05, + "loss": 0.00010419739555800334, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 526, + "tokens/total": 7941640, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 170048 + }, + { + "epoch": 8.234375, + "grad_norm": 0.0012957158032804728, + "learning_rate": 8.986311297875786e-05, + "loss": 2.4573982955189422e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 527, + "tokens/total": 7956708, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 170366 + }, + { + "epoch": 8.25, + "grad_norm": 0.0003723593254107982, + "learning_rate": 8.981713375926225e-05, + "loss": 1.1510109288792592e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 528, + "tokens/total": 7971900, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 170688 + }, + { + "epoch": 8.265625, + "grad_norm": 0.013197900727391243, + "learning_rate": 8.977106379804911e-05, + "loss": 3.1454932468477637e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 529, + "tokens/total": 7986976, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 171014 + }, + { + "epoch": 8.28125, + "grad_norm": 0.0002852350298780948, + "learning_rate": 8.97249032151877e-05, + "loss": 1.09092479760875e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 530, + "tokens/total": 8001928, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 171337 + }, + { + "epoch": 8.296875, + "grad_norm": 0.014826999045908451, + "learning_rate": 8.967865213098347e-05, + "loss": 0.00010147741704713553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 531, + "tokens/total": 8017012, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 171662 + }, + { + "epoch": 8.3125, + "grad_norm": 0.5458815693855286, + "learning_rate": 8.963231066597771e-05, + "loss": 0.005835453514009714, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00585, + "step": 532, + "tokens/total": 8032012, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 171987 + }, + { + "epoch": 8.328125, + "grad_norm": 0.001101961126551032, + "learning_rate": 8.95858789409473e-05, + "loss": 1.618544411030598e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 533, + "tokens/total": 8047000, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 172313 + }, + { + "epoch": 8.34375, + "grad_norm": 0.0038681430742144585, + "learning_rate": 8.953935707690434e-05, + "loss": 3.059169102925807e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 534, + "tokens/total": 8062088, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 172637 + }, + { + "epoch": 8.359375, + "grad_norm": 0.00867560226470232, + "learning_rate": 8.949274519509585e-05, + "loss": 3.366108285263181e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 535, + "tokens/total": 8077132, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 172961 + }, + { + "epoch": 8.375, + "grad_norm": 0.0022700580302625895, + "learning_rate": 8.944604341700346e-05, + "loss": 2.283003050251864e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 536, + "tokens/total": 8092276, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 173283 + }, + { + "epoch": 8.390625, + "grad_norm": 0.003660684684291482, + "learning_rate": 8.939925186434308e-05, + "loss": 5.109370977152139e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 537, + "tokens/total": 8107396, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 173603 + }, + { + "epoch": 8.40625, + "grad_norm": 0.005133552476763725, + "learning_rate": 8.93523706590646e-05, + "loss": 4.74297339678742e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 538, + "tokens/total": 8122408, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 173926 + }, + { + "epoch": 8.421875, + "grad_norm": 0.2128029763698578, + "learning_rate": 8.930539992335164e-05, + "loss": 0.0008521364070475101, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00085, + "step": 539, + "tokens/total": 8137452, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 174247 + }, + { + "epoch": 8.4375, + "grad_norm": 1.6053714752197266, + "learning_rate": 8.9258339779621e-05, + "loss": 0.017414025962352753, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01757, + "step": 540, + "tokens/total": 8152540, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 174573 + }, + { + "epoch": 8.453125, + "grad_norm": 0.18711458146572113, + "learning_rate": 8.921119035052265e-05, + "loss": 0.0014177690027281642, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00142, + "step": 541, + "tokens/total": 8167692, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 174898 + }, + { + "epoch": 8.46875, + "grad_norm": 0.14626744389533997, + "learning_rate": 8.916395175893916e-05, + "loss": 0.001281649456359446, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00128, + "step": 542, + "tokens/total": 8182760, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 175225 + }, + { + "epoch": 8.484375, + "grad_norm": 0.0024803634732961655, + "learning_rate": 8.911662412798553e-05, + "loss": 3.1002535251900554e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00003, + "step": 543, + "tokens/total": 8197968, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 175550 + }, + { + "epoch": 8.5, + "grad_norm": 0.004511975217610598, + "learning_rate": 8.906920758100882e-05, + "loss": 7.237551471916959e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00007, + "step": 544, + "tokens/total": 8213064, + "tokens/train_per_sec_per_gpu": 10.04, + "tokens/trainable": 175872 + }, + { + "epoch": 8.515625, + "grad_norm": 0.0012622056528925896, + "learning_rate": 8.902170224158778e-05, + "loss": 1.5453677406185307e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 545, + "tokens/total": 8228148, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 176193 + }, + { + "epoch": 8.53125, + "grad_norm": 0.001151581876911223, + "learning_rate": 8.897410823353264e-05, + "loss": 1.9299024643260054e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 546, + "tokens/total": 8243252, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 176514 + }, + { + "epoch": 8.546875, + "grad_norm": 0.949654757976532, + "learning_rate": 8.892642568088465e-05, + "loss": 0.005113059654831886, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00513, + "step": 547, + "tokens/total": 8258448, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 176839 + }, + { + "epoch": 8.5625, + "grad_norm": 0.02893899753689766, + "learning_rate": 8.887865470791589e-05, + "loss": 9.800348925637081e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 548, + "tokens/total": 8273528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 177167 + }, + { + "epoch": 8.578125, + "grad_norm": 0.3177676498889923, + "learning_rate": 8.883079543912883e-05, + "loss": 0.0007184468558989465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00072, + "step": 549, + "tokens/total": 8288496, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 177491 + }, + { + "epoch": 8.59375, + "grad_norm": 0.09711069613695145, + "learning_rate": 8.878284799925612e-05, + "loss": 0.0003404346061870456, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00034, + "step": 550, + "tokens/total": 8303696, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 177818 + }, + { + "epoch": 8.609375, + "grad_norm": 0.4529828131198883, + "learning_rate": 8.873481251326016e-05, + "loss": 0.0012388179311528802, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00124, + "step": 551, + "tokens/total": 8318700, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 178144 + }, + { + "epoch": 8.625, + "grad_norm": 0.03608764708042145, + "learning_rate": 8.868668910633283e-05, + "loss": 0.00011354131856933236, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 552, + "tokens/total": 8333792, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 178466 + }, + { + "epoch": 8.640625, + "grad_norm": 0.003121917136013508, + "learning_rate": 8.863847790389513e-05, + "loss": 2.8653050321736373e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 553, + "tokens/total": 8348860, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 178793 + }, + { + "epoch": 8.65625, + "grad_norm": 0.06226268410682678, + "learning_rate": 8.859017903159695e-05, + "loss": 0.00021325259876903147, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00021, + "step": 554, + "tokens/total": 8364084, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 179113 + }, + { + "epoch": 8.671875, + "grad_norm": 2.6193294525146484, + "learning_rate": 8.854179261531659e-05, + "loss": 0.013566155917942524, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.01366, + "step": 555, + "tokens/total": 8379180, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 179436 + }, + { + "epoch": 8.6875, + "grad_norm": 0.3264051079750061, + "learning_rate": 8.849331878116055e-05, + "loss": 0.002084167441353202, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00209, + "step": 556, + "tokens/total": 8394380, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 179753 + }, + { + "epoch": 8.703125, + "grad_norm": 0.0010522427037358284, + "learning_rate": 8.844475765546314e-05, + "loss": 1.890875500976108e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00002, + "step": 557, + "tokens/total": 8409332, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 180074 + }, + { + "epoch": 8.71875, + "grad_norm": 0.05134303867816925, + "learning_rate": 8.83961093647862e-05, + "loss": 0.00012807887105736881, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00013, + "step": 558, + "tokens/total": 8424388, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 180403 + }, + { + "epoch": 8.734375, + "grad_norm": 1.0385314226150513, + "learning_rate": 8.834737403591869e-05, + "loss": 0.02681276947259903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02718, + "step": 559, + "tokens/total": 8439436, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 180730 + }, + { + "epoch": 8.75, + "grad_norm": 0.002918509067967534, + "learning_rate": 8.82985517958765e-05, + "loss": 3.538604505592957e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 560, + "tokens/total": 8454568, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 181051 + }, + { + "epoch": 8.765625, + "grad_norm": 0.3665774464607239, + "learning_rate": 8.824964277190194e-05, + "loss": 0.0011759663466364145, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00118, + "step": 561, + "tokens/total": 8469748, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 181374 + }, + { + "epoch": 8.78125, + "grad_norm": 0.003580427495762706, + "learning_rate": 8.820064709146357e-05, + "loss": 4.611169424606487e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 562, + "tokens/total": 8484772, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 181698 + }, + { + "epoch": 8.796875, + "grad_norm": 0.06990180164575577, + "learning_rate": 8.815156488225575e-05, + "loss": 0.0006263494142331183, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00063, + "step": 563, + "tokens/total": 8499748, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 182020 + }, + { + "epoch": 8.8125, + "grad_norm": 0.0055811540223658085, + "learning_rate": 8.810239627219837e-05, + "loss": 8.284625073429197e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 564, + "tokens/total": 8514840, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 182344 + }, + { + "epoch": 8.828125, + "grad_norm": 0.35078662633895874, + "learning_rate": 8.805314138943649e-05, + "loss": 0.0007666668388992548, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00077, + "step": 565, + "tokens/total": 8529904, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 182664 + }, + { + "epoch": 8.84375, + "grad_norm": 0.0014952566707506776, + "learning_rate": 8.800380036234005e-05, + "loss": 4.0127793909050524e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 566, + "tokens/total": 8545076, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 182984 + }, + { + "epoch": 8.859375, + "grad_norm": 8.584396362304688, + "learning_rate": 8.795437331950349e-05, + "loss": 0.00995005201548338, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01, + "step": 567, + "tokens/total": 8560164, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 183306 + }, + { + "epoch": 8.875, + "grad_norm": 0.06884938478469849, + "learning_rate": 8.790486038974539e-05, + "loss": 0.00021491909865289927, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00021, + "step": 568, + "tokens/total": 8575364, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 183630 + }, + { + "epoch": 8.890625, + "grad_norm": 0.4572925865650177, + "learning_rate": 8.785526170210823e-05, + "loss": 0.03290176764130592, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03345, + "step": 569, + "tokens/total": 8590504, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 183950 + }, + { + "epoch": 8.90625, + "grad_norm": 0.0031276054214686155, + "learning_rate": 8.780557738585795e-05, + "loss": 7.353207911364734e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00007, + "step": 570, + "tokens/total": 8605536, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 184273 + }, + { + "epoch": 8.921875, + "grad_norm": 0.005127928219735622, + "learning_rate": 8.775580757048369e-05, + "loss": 0.00018162198830395937, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00018, + "step": 571, + "tokens/total": 8620496, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 184593 + }, + { + "epoch": 8.9375, + "grad_norm": 0.4080176055431366, + "learning_rate": 8.770595238569737e-05, + "loss": 0.0027535860426723957, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00276, + "step": 572, + "tokens/total": 8635532, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 184917 + }, + { + "epoch": 8.953125, + "grad_norm": 0.2424798607826233, + "learning_rate": 8.765601196143347e-05, + "loss": 0.007297150790691376, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00732, + "step": 573, + "tokens/total": 8650684, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 185241 + }, + { + "epoch": 8.96875, + "grad_norm": 0.11064363270998001, + "learning_rate": 8.76059864278486e-05, + "loss": 0.0007703031878918409, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00077, + "step": 574, + "tokens/total": 8665752, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 185565 + }, + { + "epoch": 8.984375, + "grad_norm": 0.12166444212198257, + "learning_rate": 8.755587591532116e-05, + "loss": 0.001453407690860331, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00145, + "step": 575, + "tokens/total": 8680856, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 185886 + }, + { + "epoch": 9.0, + "grad_norm": 0.01695314794778824, + "learning_rate": 8.750568055445105e-05, + "loss": 0.00043913544504903257, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00044, + "step": 576, + "tokens/total": 8695996, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 186210 + }, + { + "epoch": 9.015625, + "grad_norm": 0.03284699469804764, + "learning_rate": 8.74554004760593e-05, + "loss": 0.0009203513618558645, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00092, + "step": 577, + "tokens/total": 8711112, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 186533 + }, + { + "epoch": 9.03125, + "grad_norm": 0.2037004828453064, + "learning_rate": 8.740503581118774e-05, + "loss": 0.0039493246003985405, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00396, + "step": 578, + "tokens/total": 8726132, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 186855 + }, + { + "epoch": 9.046875, + "grad_norm": 0.015510317869484425, + "learning_rate": 8.735458669109863e-05, + "loss": 0.00044698762940242887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00045, + "step": 579, + "tokens/total": 8741288, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 187179 + }, + { + "epoch": 9.0625, + "grad_norm": 0.04965129867196083, + "learning_rate": 8.730405324727435e-05, + "loss": 0.000823804410174489, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00082, + "step": 580, + "tokens/total": 8756480, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 187503 + }, + { + "epoch": 9.078125, + "grad_norm": 0.006426906678825617, + "learning_rate": 8.72534356114171e-05, + "loss": 0.00018337769142817706, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00018, + "step": 581, + "tokens/total": 8771664, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 187824 + }, + { + "epoch": 9.09375, + "grad_norm": 0.007196735590696335, + "learning_rate": 8.720273391544839e-05, + "loss": 0.00023079977836459875, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00023, + "step": 582, + "tokens/total": 8786788, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 188147 + }, + { + "epoch": 9.109375, + "grad_norm": 0.30709800124168396, + "learning_rate": 8.715194829150892e-05, + "loss": 0.0022993457969278097, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0023, + "step": 583, + "tokens/total": 8801964, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 188471 + }, + { + "epoch": 9.125, + "grad_norm": 0.01921996846795082, + "learning_rate": 8.710107887195807e-05, + "loss": 0.00013834233686793596, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 584, + "tokens/total": 8817248, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 188799 + }, + { + "epoch": 9.140625, + "grad_norm": 0.038318585604429245, + "learning_rate": 8.705012578937364e-05, + "loss": 0.00045493399375118315, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00046, + "step": 585, + "tokens/total": 8832416, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 189119 + }, + { + "epoch": 9.15625, + "grad_norm": 0.44059014320373535, + "learning_rate": 8.699908917655145e-05, + "loss": 0.0013400256866589189, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00134, + "step": 586, + "tokens/total": 8847412, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 189437 + }, + { + "epoch": 9.171875, + "grad_norm": 0.0677911713719368, + "learning_rate": 8.694796916650503e-05, + "loss": 0.00023939728271216154, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00024, + "step": 587, + "tokens/total": 8862508, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 189761 + }, + { + "epoch": 9.1875, + "grad_norm": 0.0946345254778862, + "learning_rate": 8.689676589246528e-05, + "loss": 0.0004921150393784046, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00049, + "step": 588, + "tokens/total": 8877616, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 190085 + }, + { + "epoch": 9.203125, + "grad_norm": 0.6251227259635925, + "learning_rate": 8.684547948788008e-05, + "loss": 0.005682742223143578, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0057, + "step": 589, + "tokens/total": 8892752, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 190410 + }, + { + "epoch": 9.21875, + "grad_norm": 0.0019117683405056596, + "learning_rate": 8.679411008641397e-05, + "loss": 5.5858457926660776e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 590, + "tokens/total": 8907852, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 190732 + }, + { + "epoch": 9.234375, + "grad_norm": 0.0006620520143769681, + "learning_rate": 8.674265782194781e-05, + "loss": 2.6435986001160927e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00003, + "step": 591, + "tokens/total": 8922884, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 191054 + }, + { + "epoch": 9.25, + "grad_norm": 0.0028546350076794624, + "learning_rate": 8.66911228285784e-05, + "loss": 3.963516064686701e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00004, + "step": 592, + "tokens/total": 8937984, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 191383 + }, + { + "epoch": 9.265625, + "grad_norm": 0.0007748997304588556, + "learning_rate": 8.663950524061822e-05, + "loss": 2.5786108380998485e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 593, + "tokens/total": 8953168, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 191709 + }, + { + "epoch": 9.28125, + "grad_norm": 0.0005234112613834441, + "learning_rate": 8.658780519259493e-05, + "loss": 2.25883413804695e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 594, + "tokens/total": 8968100, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 192030 + }, + { + "epoch": 9.296875, + "grad_norm": 0.0004649423935916275, + "learning_rate": 8.653602281925113e-05, + "loss": 2.2441228793468326e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 595, + "tokens/total": 8983324, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 192352 + }, + { + "epoch": 9.3125, + "grad_norm": 0.08911292999982834, + "learning_rate": 8.648415825554399e-05, + "loss": 0.0005657169385813177, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00057, + "step": 596, + "tokens/total": 8998508, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 192675 + }, + { + "epoch": 9.328125, + "grad_norm": 0.0026829498820006847, + "learning_rate": 8.643221163664483e-05, + "loss": 3.790732444031164e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 597, + "tokens/total": 9013568, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 192995 + }, + { + "epoch": 9.34375, + "grad_norm": 0.0018017059192061424, + "learning_rate": 8.638018309793893e-05, + "loss": 2.4644470613566227e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 598, + "tokens/total": 9028760, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 193317 + }, + { + "epoch": 9.359375, + "grad_norm": 0.0754798874258995, + "learning_rate": 8.632807277502501e-05, + "loss": 0.00029164052102714777, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00029, + "step": 599, + "tokens/total": 9043940, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 193644 + }, + { + "epoch": 9.375, + "grad_norm": 0.00033408598392270505, + "learning_rate": 8.627588080371493e-05, + "loss": 1.5072258065629285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 600, + "tokens/total": 9059108, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 193969 + }, + { + "epoch": 9.390625, + "grad_norm": 0.00043498555896803737, + "learning_rate": 8.622360732003337e-05, + "loss": 1.83389875019202e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 601, + "tokens/total": 9074212, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 194291 + }, + { + "epoch": 9.40625, + "grad_norm": 0.0006206405232660472, + "learning_rate": 8.617125246021742e-05, + "loss": 2.0851886802120134e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 602, + "tokens/total": 9089180, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 194612 + }, + { + "epoch": 9.421875, + "grad_norm": 0.00028540706261992455, + "learning_rate": 8.611881636071627e-05, + "loss": 1.3558287719206419e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 603, + "tokens/total": 9104272, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 194933 + }, + { + "epoch": 9.4375, + "grad_norm": 0.0004516422050073743, + "learning_rate": 8.60662991581909e-05, + "loss": 1.6650086763547733e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 604, + "tokens/total": 9119376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 195252 + }, + { + "epoch": 9.453125, + "grad_norm": 0.002324763685464859, + "learning_rate": 8.601370098951355e-05, + "loss": 3.069010563194752e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 605, + "tokens/total": 9134396, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 195574 + }, + { + "epoch": 9.46875, + "grad_norm": 0.00039453315548598766, + "learning_rate": 8.596102199176755e-05, + "loss": 1.3730700629821513e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 606, + "tokens/total": 9149608, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 195900 + }, + { + "epoch": 9.484375, + "grad_norm": 0.0003811897477135062, + "learning_rate": 8.59082623022469e-05, + "loss": 1.661576607148163e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 607, + "tokens/total": 9164832, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 196225 + }, + { + "epoch": 9.5, + "grad_norm": 0.00027696648612618446, + "learning_rate": 8.585542205845588e-05, + "loss": 1.0936594662780408e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 608, + "tokens/total": 9179884, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 196550 + }, + { + "epoch": 9.515625, + "grad_norm": 0.00039244198706001043, + "learning_rate": 8.580250139810868e-05, + "loss": 1.2854216038249433e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 609, + "tokens/total": 9194960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 196874 + }, + { + "epoch": 9.53125, + "grad_norm": 0.00029694641125388443, + "learning_rate": 8.57495004591291e-05, + "loss": 1.0953734090435319e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 610, + "tokens/total": 9210108, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 197195 + }, + { + "epoch": 9.546875, + "grad_norm": 0.0006175542366690934, + "learning_rate": 8.56964193796502e-05, + "loss": 1.0757680684037041e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 611, + "tokens/total": 9225252, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 197519 + }, + { + "epoch": 9.5625, + "grad_norm": 0.00114785460755229, + "learning_rate": 8.56432582980139e-05, + "loss": 1.7286893125856295e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 612, + "tokens/total": 9240320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 197843 + }, + { + "epoch": 9.578125, + "grad_norm": 0.0005527856992557645, + "learning_rate": 8.559001735277055e-05, + "loss": 1.5963505575200543e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 613, + "tokens/total": 9255456, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 198165 + }, + { + "epoch": 9.59375, + "grad_norm": 0.00021238153567537665, + "learning_rate": 8.553669668267871e-05, + "loss": 7.194825684564421e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 614, + "tokens/total": 9270556, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 198486 + }, + { + "epoch": 9.609375, + "grad_norm": 0.0012781359255313873, + "learning_rate": 8.548329642670472e-05, + "loss": 2.0431305529200472e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 615, + "tokens/total": 9285660, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 198814 + }, + { + "epoch": 9.625, + "grad_norm": 0.00020792640862055123, + "learning_rate": 8.542981672402233e-05, + "loss": 7.524381999246543e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 616, + "tokens/total": 9300668, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 199136 + }, + { + "epoch": 9.640625, + "grad_norm": 0.0003108520177192986, + "learning_rate": 8.537625771401231e-05, + "loss": 1.0943025699816644e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 617, + "tokens/total": 9315668, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 199460 + }, + { + "epoch": 9.65625, + "grad_norm": 0.0002239548193756491, + "learning_rate": 8.53226195362622e-05, + "loss": 9.883589882520027e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 618, + "tokens/total": 9330848, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 199788 + }, + { + "epoch": 9.671875, + "grad_norm": 0.00030563052860088646, + "learning_rate": 8.526890233056579e-05, + "loss": 9.192094694299158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 619, + "tokens/total": 9345924, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 200110 + }, + { + "epoch": 9.6875, + "grad_norm": 0.0312703438103199, + "learning_rate": 8.521510623692288e-05, + "loss": 0.00012628451804630458, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 620, + "tokens/total": 9361068, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 200438 + }, + { + "epoch": 9.703125, + "grad_norm": 0.00016913052240852267, + "learning_rate": 8.516123139553889e-05, + "loss": 6.865585874038516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 621, + "tokens/total": 9376068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 200762 + }, + { + "epoch": 9.71875, + "grad_norm": 0.00023767122183926404, + "learning_rate": 8.510727794682443e-05, + "loss": 9.91942852124339e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 622, + "tokens/total": 9391068, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 201087 + }, + { + "epoch": 9.734375, + "grad_norm": 0.0003941507893614471, + "learning_rate": 8.505324603139499e-05, + "loss": 1.4861869203741662e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 623, + "tokens/total": 9406204, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 201407 + }, + { + "epoch": 9.75, + "grad_norm": 0.00018824409926310182, + "learning_rate": 8.499913579007058e-05, + "loss": 8.226126738009043e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 624, + "tokens/total": 9421308, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 201727 + }, + { + "epoch": 9.765625, + "grad_norm": 0.0001939542853506282, + "learning_rate": 8.494494736387535e-05, + "loss": 8.253552550741006e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 625, + "tokens/total": 9436284, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 202052 + }, + { + "epoch": 9.78125, + "grad_norm": 0.0003432348894421011, + "learning_rate": 8.489068089403719e-05, + "loss": 1.0490179192856885e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 626, + "tokens/total": 9451392, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 202375 + }, + { + "epoch": 9.796875, + "grad_norm": 0.0004783475014846772, + "learning_rate": 8.483633652198739e-05, + "loss": 1.068074743670877e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 627, + "tokens/total": 9466440, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 202706 + }, + { + "epoch": 9.8125, + "grad_norm": 0.00028024587663821876, + "learning_rate": 8.47819143893603e-05, + "loss": 9.895077710098121e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 628, + "tokens/total": 9481532, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 203028 + }, + { + "epoch": 9.828125, + "grad_norm": 0.00017581306747160852, + "learning_rate": 8.472741463799296e-05, + "loss": 6.723947535647312e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 629, + "tokens/total": 9496764, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 203351 + }, + { + "epoch": 9.84375, + "grad_norm": 0.00013647547166328877, + "learning_rate": 8.467283740992458e-05, + "loss": 5.100245743960841e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 630, + "tokens/total": 9511892, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 203673 + }, + { + "epoch": 9.859375, + "grad_norm": 0.00022224783606361598, + "learning_rate": 8.461818284739645e-05, + "loss": 7.700567948631942e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 631, + "tokens/total": 9526912, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 203992 + }, + { + "epoch": 9.875, + "grad_norm": 0.033552926033735275, + "learning_rate": 8.456345109285131e-05, + "loss": 7.867354725021869e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 632, + "tokens/total": 9542048, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 204316 + }, + { + "epoch": 9.890625, + "grad_norm": 0.00016879606118891388, + "learning_rate": 8.45086422889331e-05, + "loss": 6.822014256613329e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 633, + "tokens/total": 9557160, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 204640 + }, + { + "epoch": 9.90625, + "grad_norm": 0.0007644848665222526, + "learning_rate": 8.445375657848657e-05, + "loss": 1.4742925486643799e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 634, + "tokens/total": 9572248, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 204962 + }, + { + "epoch": 9.921875, + "grad_norm": 0.00015245663234964013, + "learning_rate": 8.439879410455693e-05, + "loss": 6.433082944568014e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 635, + "tokens/total": 9587332, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 205286 + }, + { + "epoch": 9.9375, + "grad_norm": 0.00024901528377085924, + "learning_rate": 8.434375501038944e-05, + "loss": 7.426990123349242e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 636, + "tokens/total": 9602432, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 205611 + }, + { + "epoch": 9.953125, + "grad_norm": 0.00020785727247130126, + "learning_rate": 8.428863943942904e-05, + "loss": 4.663966592488578e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 637, + "tokens/total": 9617432, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 205935 + }, + { + "epoch": 9.96875, + "grad_norm": 0.0003147262614220381, + "learning_rate": 8.423344753531999e-05, + "loss": 1.0147239663638175e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 638, + "tokens/total": 9632656, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 206254 + }, + { + "epoch": 9.984375, + "grad_norm": 0.00011762624490074813, + "learning_rate": 8.417817944190553e-05, + "loss": 5.135711944603827e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 639, + "tokens/total": 9647680, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 206574 + }, + { + "epoch": 10.0, + "grad_norm": 0.00015940859157126397, + "learning_rate": 8.412283530322736e-05, + "loss": 6.558111181220738e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 640, + "tokens/total": 9662672, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 206900 + }, + { + "epoch": 10.015625, + "grad_norm": 0.00020247366046532989, + "learning_rate": 8.406741526352553e-05, + "loss": 7.633244422322605e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 641, + "tokens/total": 9677748, + "tokens/train_per_sec_per_gpu": 8.85, + "tokens/trainable": 207225 + }, + { + "epoch": 10.03125, + "grad_norm": 0.0004867524839937687, + "learning_rate": 8.401191946723776e-05, + "loss": 1.1116100722574629e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 642, + "tokens/total": 9692824, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 207549 + }, + { + "epoch": 10.046875, + "grad_norm": 0.00011772097059292719, + "learning_rate": 8.395634805899931e-05, + "loss": 4.867678399023134e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 643, + "tokens/total": 9707908, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 207868 + }, + { + "epoch": 10.0625, + "grad_norm": 0.00015480454021599144, + "learning_rate": 8.390070118364246e-05, + "loss": 5.00797386848717e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 644, + "tokens/total": 9722956, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 208190 + }, + { + "epoch": 10.078125, + "grad_norm": 0.00012757439981214702, + "learning_rate": 8.384497898619617e-05, + "loss": 5.3190769904176705e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 645, + "tokens/total": 9738088, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 208514 + }, + { + "epoch": 10.09375, + "grad_norm": 0.0001416898739989847, + "learning_rate": 8.378918161188571e-05, + "loss": 5.836336185893742e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 646, + "tokens/total": 9753164, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 208839 + }, + { + "epoch": 10.109375, + "grad_norm": 0.00010145518172066659, + "learning_rate": 8.373330920613229e-05, + "loss": 4.150137101532891e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 647, + "tokens/total": 9768176, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 209161 + }, + { + "epoch": 10.125, + "grad_norm": 0.0002767986443359405, + "learning_rate": 8.367736191455268e-05, + "loss": 6.726240826537833e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 648, + "tokens/total": 9783276, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 209483 + }, + { + "epoch": 10.140625, + "grad_norm": 0.00012657605111598969, + "learning_rate": 8.362133988295879e-05, + "loss": 5.29480712430086e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 649, + "tokens/total": 9798340, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 209810 + }, + { + "epoch": 10.15625, + "grad_norm": 0.00016513089940417558, + "learning_rate": 8.356524325735735e-05, + "loss": 4.137089945288608e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 650, + "tokens/total": 9813404, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 210132 + }, + { + "epoch": 10.171875, + "grad_norm": 0.00013624246639665216, + "learning_rate": 8.35090721839495e-05, + "loss": 5.879107447981369e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 651, + "tokens/total": 9828520, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 210459 + }, + { + "epoch": 10.1875, + "grad_norm": 0.00032070872839540243, + "learning_rate": 8.345282680913033e-05, + "loss": 7.33726710677729e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 652, + "tokens/total": 9843532, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 210785 + }, + { + "epoch": 10.203125, + "grad_norm": 0.00013062500511296093, + "learning_rate": 8.33965072794887e-05, + "loss": 5.269500888971379e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 653, + "tokens/total": 9858592, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 211109 + }, + { + "epoch": 10.21875, + "grad_norm": 0.000225347641389817, + "learning_rate": 8.334011374180668e-05, + "loss": 6.59221041132696e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 654, + "tokens/total": 9873660, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 211432 + }, + { + "epoch": 10.234375, + "grad_norm": 0.00013657586532644928, + "learning_rate": 8.328364634305917e-05, + "loss": 5.456272447190713e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 655, + "tokens/total": 9888736, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 211754 + }, + { + "epoch": 10.25, + "grad_norm": 3.6042065620422363, + "learning_rate": 8.322710523041366e-05, + "loss": 0.014105875976383686, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01421, + "step": 656, + "tokens/total": 9903756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 212075 + }, + { + "epoch": 10.265625, + "grad_norm": 0.00015812524361535907, + "learning_rate": 8.317049055122971e-05, + "loss": 5.314343525242293e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 657, + "tokens/total": 9918800, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 212399 + }, + { + "epoch": 10.28125, + "grad_norm": 0.00023147545289248228, + "learning_rate": 8.311380245305863e-05, + "loss": 6.949136150069535e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 658, + "tokens/total": 9933768, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 212718 + }, + { + "epoch": 10.296875, + "grad_norm": 0.00014753777941223234, + "learning_rate": 8.305704108364301e-05, + "loss": 5.799164227937581e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 659, + "tokens/total": 9948864, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 213040 + }, + { + "epoch": 10.3125, + "grad_norm": 0.0011448130244389176, + "learning_rate": 8.30002065909165e-05, + "loss": 2.2385131160262972e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 660, + "tokens/total": 9964024, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 213361 + }, + { + "epoch": 10.328125, + "grad_norm": 0.11430440843105316, + "learning_rate": 8.294329912300329e-05, + "loss": 0.0007114034378901124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00071, + "step": 661, + "tokens/total": 9979200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 213682 + }, + { + "epoch": 10.34375, + "grad_norm": 1.3747159242630005, + "learning_rate": 8.288631882821773e-05, + "loss": 0.007618979085236788, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00765, + "step": 662, + "tokens/total": 9994184, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 214002 + }, + { + "epoch": 10.359375, + "grad_norm": 0.0006237569032236934, + "learning_rate": 8.282926585506399e-05, + "loss": 9.479489563091192e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 663, + "tokens/total": 10009244, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 214318 + }, + { + "epoch": 10.375, + "grad_norm": 0.0012571830302476883, + "learning_rate": 8.277214035223568e-05, + "loss": 1.839931974245701e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 664, + "tokens/total": 10024264, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 214647 + }, + { + "epoch": 10.390625, + "grad_norm": 0.33910050988197327, + "learning_rate": 8.271494246861545e-05, + "loss": 0.0007400690228678286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00074, + "step": 665, + "tokens/total": 10039468, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 214968 + }, + { + "epoch": 10.40625, + "grad_norm": 0.0007855923031456769, + "learning_rate": 8.26576723532745e-05, + "loss": 1.8088217984768562e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 666, + "tokens/total": 10054640, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 215291 + }, + { + "epoch": 10.421875, + "grad_norm": 0.17555910348892212, + "learning_rate": 8.260033015547241e-05, + "loss": 0.0004138879885431379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00041, + "step": 667, + "tokens/total": 10069820, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 215612 + }, + { + "epoch": 10.4375, + "grad_norm": 0.0006887784111313522, + "learning_rate": 8.254291602465653e-05, + "loss": 1.740579136821907e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 668, + "tokens/total": 10084928, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 215940 + }, + { + "epoch": 10.453125, + "grad_norm": 0.0003940793394576758, + "learning_rate": 8.248543011046174e-05, + "loss": 1.1545018423930742e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 669, + "tokens/total": 10099972, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 216264 + }, + { + "epoch": 10.46875, + "grad_norm": 0.0041761621832847595, + "learning_rate": 8.242787256270995e-05, + "loss": 4.01059805881232e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00004, + "step": 670, + "tokens/total": 10115120, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 216588 + }, + { + "epoch": 10.484375, + "grad_norm": 0.006432351656258106, + "learning_rate": 8.237024353140983e-05, + "loss": 4.4539981900015846e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 671, + "tokens/total": 10130164, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 216911 + }, + { + "epoch": 10.5, + "grad_norm": 0.008654266595840454, + "learning_rate": 8.231254316675629e-05, + "loss": 9.718702494865283e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 672, + "tokens/total": 10145220, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 217239 + }, + { + "epoch": 10.515625, + "grad_norm": 0.0033223514910787344, + "learning_rate": 8.22547716191302e-05, + "loss": 2.6446699848747812e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 673, + "tokens/total": 10160416, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 217566 + }, + { + "epoch": 10.53125, + "grad_norm": 0.0029003892559558153, + "learning_rate": 8.219692903909794e-05, + "loss": 4.9929614760912955e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 674, + "tokens/total": 10175636, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 217890 + }, + { + "epoch": 10.546875, + "grad_norm": 0.0760011374950409, + "learning_rate": 8.213901557741098e-05, + "loss": 0.0007359297596849501, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00074, + "step": 675, + "tokens/total": 10190752, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 218218 + }, + { + "epoch": 10.5625, + "grad_norm": 0.2406068593263626, + "learning_rate": 8.208103138500557e-05, + "loss": 0.0019148560240864754, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00192, + "step": 676, + "tokens/total": 10205880, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 218539 + }, + { + "epoch": 10.578125, + "grad_norm": 0.0008238900336436927, + "learning_rate": 8.202297661300228e-05, + "loss": 1.983395304705482e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 677, + "tokens/total": 10220984, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 218861 + }, + { + "epoch": 10.59375, + "grad_norm": 0.0006221993826329708, + "learning_rate": 8.196485141270563e-05, + "loss": 1.3678405593964271e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 678, + "tokens/total": 10236212, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 219186 + }, + { + "epoch": 10.609375, + "grad_norm": 0.0014093968784436584, + "learning_rate": 8.190665593560367e-05, + "loss": 1.4763102626602631e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 679, + "tokens/total": 10251296, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 219510 + }, + { + "epoch": 10.625, + "grad_norm": 0.4953056871891022, + "learning_rate": 8.184839033336767e-05, + "loss": 0.015656938776373863, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01578, + "step": 680, + "tokens/total": 10266344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 219834 + }, + { + "epoch": 10.640625, + "grad_norm": 0.0181986466050148, + "learning_rate": 8.179005475785162e-05, + "loss": 0.00013063072401564568, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 681, + "tokens/total": 10281380, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 220156 + }, + { + "epoch": 10.65625, + "grad_norm": 0.008455532602965832, + "learning_rate": 8.173164936109184e-05, + "loss": 0.00016575312474742532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00017, + "step": 682, + "tokens/total": 10296504, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 220479 + }, + { + "epoch": 10.671875, + "grad_norm": 0.0012320041423663497, + "learning_rate": 8.167317429530669e-05, + "loss": 3.3237243769690394e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 683, + "tokens/total": 10311544, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 220803 + }, + { + "epoch": 10.6875, + "grad_norm": 0.00673325639218092, + "learning_rate": 8.161462971289608e-05, + "loss": 0.00015895556134637445, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00016, + "step": 684, + "tokens/total": 10326644, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 221124 + }, + { + "epoch": 10.703125, + "grad_norm": 0.1733241081237793, + "learning_rate": 8.155601576644108e-05, + "loss": 0.0025445870123803616, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00255, + "step": 685, + "tokens/total": 10341824, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 221449 + }, + { + "epoch": 10.71875, + "grad_norm": 0.021975671872496605, + "learning_rate": 8.14973326087036e-05, + "loss": 0.0004545206611510366, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 686, + "tokens/total": 10357076, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 221774 + }, + { + "epoch": 10.734375, + "grad_norm": 0.21455904841423035, + "learning_rate": 8.14385803926258e-05, + "loss": 0.004639237187802792, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00465, + "step": 687, + "tokens/total": 10372192, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 222094 + }, + { + "epoch": 10.75, + "grad_norm": 0.028662165626883507, + "learning_rate": 8.137975927132995e-05, + "loss": 0.0003142151399515569, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 688, + "tokens/total": 10387336, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 222419 + }, + { + "epoch": 10.765625, + "grad_norm": 0.0500834695994854, + "learning_rate": 8.132086939811787e-05, + "loss": 0.0010720632271841168, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00107, + "step": 689, + "tokens/total": 10402484, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 222742 + }, + { + "epoch": 10.78125, + "grad_norm": 0.04645442217588425, + "learning_rate": 8.126191092647053e-05, + "loss": 0.00026277147117070854, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00026, + "step": 690, + "tokens/total": 10417640, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 223062 + }, + { + "epoch": 10.796875, + "grad_norm": 0.007354910485446453, + "learning_rate": 8.120288401004771e-05, + "loss": 0.00011086190352216363, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00011, + "step": 691, + "tokens/total": 10432700, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 223383 + }, + { + "epoch": 10.8125, + "grad_norm": 0.00775782810524106, + "learning_rate": 8.114378880268758e-05, + "loss": 0.0001411090634064749, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 692, + "tokens/total": 10447872, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 223707 + }, + { + "epoch": 10.828125, + "grad_norm": 0.0007672952488064766, + "learning_rate": 8.108462545840627e-05, + "loss": 2.438546289340593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 693, + "tokens/total": 10462960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 224031 + }, + { + "epoch": 10.84375, + "grad_norm": 0.03337325155735016, + "learning_rate": 8.102539413139751e-05, + "loss": 0.0002755741588771343, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 694, + "tokens/total": 10477964, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 224352 + }, + { + "epoch": 10.859375, + "grad_norm": 0.008774218149483204, + "learning_rate": 8.09660949760322e-05, + "loss": 7.373647531494498e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00007, + "step": 695, + "tokens/total": 10493088, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 224677 + }, + { + "epoch": 10.875, + "grad_norm": 0.19777068495750427, + "learning_rate": 8.090672814685802e-05, + "loss": 0.002209917875006795, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00221, + "step": 696, + "tokens/total": 10508160, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 225001 + }, + { + "epoch": 10.890625, + "grad_norm": 0.0007365435594692826, + "learning_rate": 8.084729379859902e-05, + "loss": 2.1141760953469202e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 697, + "tokens/total": 10523108, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 225323 + }, + { + "epoch": 10.90625, + "grad_norm": 0.0004815941210836172, + "learning_rate": 8.078779208615524e-05, + "loss": 1.7367412510793656e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 698, + "tokens/total": 10538160, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 225645 + }, + { + "epoch": 10.921875, + "grad_norm": 0.00034571634023450315, + "learning_rate": 8.072822316460223e-05, + "loss": 1.359562884317711e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 699, + "tokens/total": 10553296, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 225974 + }, + { + "epoch": 10.9375, + "grad_norm": 0.04475097730755806, + "learning_rate": 8.066858718919078e-05, + "loss": 0.0004733581154141575, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00047, + "step": 700, + "tokens/total": 10568452, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 226296 + }, + { + "epoch": 10.953125, + "grad_norm": 0.00406939210370183, + "learning_rate": 8.060888431534638e-05, + "loss": 4.0564369555795565e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 701, + "tokens/total": 10583552, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 226619 + }, + { + "epoch": 10.96875, + "grad_norm": 0.0006428882479667664, + "learning_rate": 8.054911469866894e-05, + "loss": 1.8578755771159194e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 702, + "tokens/total": 10598668, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 226944 + }, + { + "epoch": 10.984375, + "grad_norm": 0.0004231981001794338, + "learning_rate": 8.048927849493222e-05, + "loss": 1.3509456039173529e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00001, + "step": 703, + "tokens/total": 10613828, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 227269 + }, + { + "epoch": 11.0, + "grad_norm": 0.0003220014041289687, + "learning_rate": 8.042937586008359e-05, + "loss": 1.1975650522799697e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 704, + "tokens/total": 10628960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 227590 + }, + { + "epoch": 11.015625, + "grad_norm": 0.0004653835785575211, + "learning_rate": 8.036940695024354e-05, + "loss": 1.2614174011105206e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 705, + "tokens/total": 10644044, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 227915 + }, + { + "epoch": 11.03125, + "grad_norm": 0.0010091683361679316, + "learning_rate": 8.030937192170529e-05, + "loss": 2.3527491066488437e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 706, + "tokens/total": 10659112, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 228237 + }, + { + "epoch": 11.046875, + "grad_norm": 0.0002630383532959968, + "learning_rate": 8.02492709309344e-05, + "loss": 1.0589868907118216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 707, + "tokens/total": 10674172, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 228567 + }, + { + "epoch": 11.0625, + "grad_norm": 0.0003388511249795556, + "learning_rate": 8.018910413456828e-05, + "loss": 9.929903171723709e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 708, + "tokens/total": 10689384, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 228891 + }, + { + "epoch": 11.078125, + "grad_norm": 0.00021411596389953047, + "learning_rate": 8.012887168941592e-05, + "loss": 9.911593224387616e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 709, + "tokens/total": 10704556, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 229221 + }, + { + "epoch": 11.09375, + "grad_norm": 0.00035079781082458794, + "learning_rate": 8.006857375245736e-05, + "loss": 1.117552073992556e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 710, + "tokens/total": 10719640, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 229540 + }, + { + "epoch": 11.109375, + "grad_norm": 0.00019558356143534184, + "learning_rate": 8.000821048084334e-05, + "loss": 8.218433322326746e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 711, + "tokens/total": 10734776, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 229861 + }, + { + "epoch": 11.125, + "grad_norm": 0.0003149871772620827, + "learning_rate": 7.994778203189489e-05, + "loss": 8.92526895768242e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 712, + "tokens/total": 10749824, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 230186 + }, + { + "epoch": 11.140625, + "grad_norm": 0.00013192363257985562, + "learning_rate": 7.988728856310289e-05, + "loss": 4.4209923544258345e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 713, + "tokens/total": 10764920, + "tokens/train_per_sec_per_gpu": 8.44, + "tokens/trainable": 230511 + }, + { + "epoch": 11.15625, + "grad_norm": 0.004311064258217812, + "learning_rate": 7.982673023212767e-05, + "loss": 3.988208118244074e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 714, + "tokens/total": 10780068, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 230834 + }, + { + "epoch": 11.171875, + "grad_norm": 0.0002725148806348443, + "learning_rate": 7.976610719679865e-05, + "loss": 8.843769137456547e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 715, + "tokens/total": 10795172, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 231159 + }, + { + "epoch": 11.1875, + "grad_norm": 0.0005816842312924564, + "learning_rate": 7.970541961511382e-05, + "loss": 1.673822225711774e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 716, + "tokens/total": 10810260, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 231480 + }, + { + "epoch": 11.203125, + "grad_norm": 0.0003033979737665504, + "learning_rate": 7.964466764523945e-05, + "loss": 1.0691360330383759e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 717, + "tokens/total": 10825452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 231806 + }, + { + "epoch": 11.21875, + "grad_norm": 0.00017789554840419441, + "learning_rate": 7.958385144550958e-05, + "loss": 6.68784741719719e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 718, + "tokens/total": 10840596, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 232128 + }, + { + "epoch": 11.234375, + "grad_norm": 0.00024442930589430034, + "learning_rate": 7.95229711744257e-05, + "loss": 8.834139407554176e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 719, + "tokens/total": 10855700, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 232446 + }, + { + "epoch": 11.25, + "grad_norm": 0.0009943818440660834, + "learning_rate": 7.946202699065622e-05, + "loss": 1.56411206262419e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 720, + "tokens/total": 10870892, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 232771 + }, + { + "epoch": 11.265625, + "grad_norm": 0.00025015894789248705, + "learning_rate": 7.940101905303615e-05, + "loss": 8.422994142165408e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 721, + "tokens/total": 10885960, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 233096 + }, + { + "epoch": 11.28125, + "grad_norm": 0.000299341743811965, + "learning_rate": 7.93399475205667e-05, + "loss": 7.969428224896546e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 722, + "tokens/total": 10900976, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 233421 + }, + { + "epoch": 11.296875, + "grad_norm": 0.00023623707238584757, + "learning_rate": 7.927881255241474e-05, + "loss": 8.238984264608007e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 723, + "tokens/total": 10916092, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 233746 + }, + { + "epoch": 11.3125, + "grad_norm": 0.0009587532840669155, + "learning_rate": 7.921761430791252e-05, + "loss": 6.994597242737655e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 724, + "tokens/total": 10931228, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 234068 + }, + { + "epoch": 11.328125, + "grad_norm": 0.0006762287230230868, + "learning_rate": 7.915635294655723e-05, + "loss": 8.578419510740787e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 725, + "tokens/total": 10946452, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 234389 + }, + { + "epoch": 11.34375, + "grad_norm": 0.0001469070848543197, + "learning_rate": 7.909502862801049e-05, + "loss": 4.493551841733279e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 726, + "tokens/total": 10961580, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 234715 + }, + { + "epoch": 11.359375, + "grad_norm": 0.000356843724148348, + "learning_rate": 7.903364151209804e-05, + "loss": 8.137749318848364e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 727, + "tokens/total": 10976596, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 235034 + }, + { + "epoch": 11.375, + "grad_norm": 0.00029630697099491954, + "learning_rate": 7.897219175880931e-05, + "loss": 8.473220077576116e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 728, + "tokens/total": 10991728, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 235355 + }, + { + "epoch": 11.390625, + "grad_norm": 0.004472525790333748, + "learning_rate": 7.89106795282969e-05, + "loss": 1.0785695849335752e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 729, + "tokens/total": 11006712, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 235674 + }, + { + "epoch": 11.40625, + "grad_norm": 0.00018268785788677633, + "learning_rate": 7.884910498087632e-05, + "loss": 5.513010819413466e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 730, + "tokens/total": 11021724, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 235999 + }, + { + "epoch": 11.421875, + "grad_norm": 0.0002784350363072008, + "learning_rate": 7.878746827702545e-05, + "loss": 6.694717285427032e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 731, + "tokens/total": 11036780, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 236322 + }, + { + "epoch": 11.4375, + "grad_norm": 0.0005618222639895976, + "learning_rate": 7.872576957738418e-05, + "loss": 1.0616214240144473e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 732, + "tokens/total": 11051868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 236642 + }, + { + "epoch": 11.453125, + "grad_norm": 0.00024973825202323496, + "learning_rate": 7.866400904275396e-05, + "loss": 9.546066394250374e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 733, + "tokens/total": 11066984, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 236964 + }, + { + "epoch": 11.46875, + "grad_norm": 0.00027185113867744803, + "learning_rate": 7.860218683409742e-05, + "loss": 9.028317435877398e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 734, + "tokens/total": 11082000, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 237291 + }, + { + "epoch": 11.484375, + "grad_norm": 0.00012226897524669766, + "learning_rate": 7.854030311253792e-05, + "loss": 4.406361313158413e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 735, + "tokens/total": 11097164, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 237614 + }, + { + "epoch": 11.5, + "grad_norm": 0.0001397662708768621, + "learning_rate": 7.847835803935911e-05, + "loss": 4.8795532165968325e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 736, + "tokens/total": 11112288, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 237939 + }, + { + "epoch": 11.515625, + "grad_norm": 0.0012317969230934978, + "learning_rate": 7.841635177600457e-05, + "loss": 1.585951031302102e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 737, + "tokens/total": 11127476, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 238268 + }, + { + "epoch": 11.53125, + "grad_norm": 0.00045033343485556543, + "learning_rate": 7.835428448407737e-05, + "loss": 6.74296825309284e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 738, + "tokens/total": 11142488, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 238592 + }, + { + "epoch": 11.546875, + "grad_norm": 0.00011499993706820533, + "learning_rate": 7.829215632533957e-05, + "loss": 3.6553283280227333e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 739, + "tokens/total": 11157532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 238916 + }, + { + "epoch": 11.5625, + "grad_norm": 0.00022261161939240992, + "learning_rate": 7.822996746171192e-05, + "loss": 8.351025826414116e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 740, + "tokens/total": 11172620, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 239235 + }, + { + "epoch": 11.578125, + "grad_norm": 0.0009697281057015061, + "learning_rate": 7.816771805527336e-05, + "loss": 1.1542260835994966e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 741, + "tokens/total": 11187780, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 239557 + }, + { + "epoch": 11.59375, + "grad_norm": 0.00017928672605194151, + "learning_rate": 7.810540826826064e-05, + "loss": 6.871232017147122e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 742, + "tokens/total": 11202908, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 239884 + }, + { + "epoch": 11.609375, + "grad_norm": 0.00015312970208469778, + "learning_rate": 7.804303826306784e-05, + "loss": 4.4953990254725795e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 743, + "tokens/total": 11218028, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 240200 + }, + { + "epoch": 11.625, + "grad_norm": 0.00021180683688726276, + "learning_rate": 7.798060820224602e-05, + "loss": 7.936658221296966e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 744, + "tokens/total": 11233028, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 240521 + }, + { + "epoch": 11.640625, + "grad_norm": 0.00026271751266904175, + "learning_rate": 7.791811824850271e-05, + "loss": 6.429564564314205e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 745, + "tokens/total": 11248200, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 240844 + }, + { + "epoch": 11.65625, + "grad_norm": 0.000112937756057363, + "learning_rate": 7.78555685647016e-05, + "loss": 4.1820148908300325e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 746, + "tokens/total": 11263204, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 241171 + }, + { + "epoch": 11.671875, + "grad_norm": 0.000401324505219236, + "learning_rate": 7.779295931386202e-05, + "loss": 5.586419320025016e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 747, + "tokens/total": 11278288, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 241493 + }, + { + "epoch": 11.6875, + "grad_norm": 0.0013540866784751415, + "learning_rate": 7.773029065915851e-05, + "loss": 1.639010224607773e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 748, + "tokens/total": 11293364, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 241821 + }, + { + "epoch": 11.703125, + "grad_norm": 0.0008356674807146192, + "learning_rate": 7.766756276392049e-05, + "loss": 4.819988134840969e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 749, + "tokens/total": 11308524, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 242145 + }, + { + "epoch": 11.71875, + "grad_norm": 0.00012949411757290363, + "learning_rate": 7.760477579163175e-05, + "loss": 4.385303327580914e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 750, + "tokens/total": 11323636, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 242475 + }, + { + "epoch": 11.734375, + "grad_norm": 0.00033964822068810463, + "learning_rate": 7.754192990593006e-05, + "loss": 7.976723281899467e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 751, + "tokens/total": 11338712, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 242796 + }, + { + "epoch": 11.75, + "grad_norm": 0.0003038443101104349, + "learning_rate": 7.747902527060669e-05, + "loss": 7.84312123869313e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 752, + "tokens/total": 11353776, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 243120 + }, + { + "epoch": 11.765625, + "grad_norm": 0.00036285549867898226, + "learning_rate": 7.741606204960606e-05, + "loss": 1.0751992704172153e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 753, + "tokens/total": 11368904, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 243445 + }, + { + "epoch": 11.78125, + "grad_norm": 0.00016507648979313672, + "learning_rate": 7.735304040702528e-05, + "loss": 5.627839527733158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 754, + "tokens/total": 11384068, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 243769 + }, + { + "epoch": 11.796875, + "grad_norm": 0.00013171533646527678, + "learning_rate": 7.728996050711372e-05, + "loss": 4.834202172787627e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 755, + "tokens/total": 11399116, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 244090 + }, + { + "epoch": 11.8125, + "grad_norm": 0.00026781720225699246, + "learning_rate": 7.722682251427256e-05, + "loss": 6.27858889856725e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 756, + "tokens/total": 11414248, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 244411 + }, + { + "epoch": 11.828125, + "grad_norm": 0.00013672966451849788, + "learning_rate": 7.716362659305444e-05, + "loss": 3.8341331674018875e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 757, + "tokens/total": 11429368, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 244739 + }, + { + "epoch": 11.84375, + "grad_norm": 0.00018275645561516285, + "learning_rate": 7.710037290816289e-05, + "loss": 5.9024632719228975e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 758, + "tokens/total": 11444340, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 245060 + }, + { + "epoch": 11.859375, + "grad_norm": 0.00012778387463185936, + "learning_rate": 7.703706162445204e-05, + "loss": 4.635352070181398e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 759, + "tokens/total": 11459428, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 245383 + }, + { + "epoch": 11.875, + "grad_norm": 0.00010651875345502049, + "learning_rate": 7.697369290692617e-05, + "loss": 3.822206508630188e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 760, + "tokens/total": 11474592, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 245704 + }, + { + "epoch": 11.890625, + "grad_norm": 0.00013545139518100768, + "learning_rate": 7.691026692073915e-05, + "loss": 4.4806192818214186e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 761, + "tokens/total": 11489644, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 246026 + }, + { + "epoch": 11.90625, + "grad_norm": 0.001958167180418968, + "learning_rate": 7.68467838311942e-05, + "loss": 2.2847019863547757e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 762, + "tokens/total": 11504716, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 246349 + }, + { + "epoch": 11.921875, + "grad_norm": 0.00014885167183820158, + "learning_rate": 7.678324380374328e-05, + "loss": 3.5454936551104765e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 763, + "tokens/total": 11519828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 246668 + }, + { + "epoch": 11.9375, + "grad_norm": 0.00020161131396889687, + "learning_rate": 7.671964700398684e-05, + "loss": 5.791897820017766e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 764, + "tokens/total": 11534864, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 246991 + }, + { + "epoch": 11.953125, + "grad_norm": 0.00011139918206026778, + "learning_rate": 7.66559935976732e-05, + "loss": 3.7280333344824612e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 765, + "tokens/total": 11549908, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 247312 + }, + { + "epoch": 11.96875, + "grad_norm": 0.00012893143866676837, + "learning_rate": 7.659228375069827e-05, + "loss": 4.458348030311754e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 766, + "tokens/total": 11565044, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 247636 + }, + { + "epoch": 11.984375, + "grad_norm": 0.00016953289741650224, + "learning_rate": 7.652851762910505e-05, + "loss": 3.7684642393287504e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 767, + "tokens/total": 11580188, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 247959 + }, + { + "epoch": 12.0, + "grad_norm": 0.00011622429155977443, + "learning_rate": 7.646469539908316e-05, + "loss": 3.689345021484769e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 768, + "tokens/total": 11595328, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 248280 + }, + { + "epoch": 12.015625, + "grad_norm": 0.0010329362703487277, + "learning_rate": 7.640081722696851e-05, + "loss": 1.379754212393891e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 769, + "tokens/total": 11610408, + "tokens/train_per_sec_per_gpu": 8.75, + "tokens/trainable": 248603 + }, + { + "epoch": 12.03125, + "grad_norm": 8.753351721679792e-05, + "learning_rate": 7.633688327924279e-05, + "loss": 3.1169097383099142e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 770, + "tokens/total": 11625440, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 248929 + }, + { + "epoch": 12.046875, + "grad_norm": 0.00015718373470008373, + "learning_rate": 7.627289372253304e-05, + "loss": 4.225745669828029e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 771, + "tokens/total": 11640488, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 249250 + }, + { + "epoch": 12.0625, + "grad_norm": 0.00012905069161206484, + "learning_rate": 7.620884872361122e-05, + "loss": 4.561928562907269e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 772, + "tokens/total": 11655628, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 249573 + }, + { + "epoch": 12.078125, + "grad_norm": 0.000239390748902224, + "learning_rate": 7.61447484493938e-05, + "loss": 4.851561243413016e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 773, + "tokens/total": 11670680, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 249895 + }, + { + "epoch": 12.09375, + "grad_norm": 0.00010798968287417665, + "learning_rate": 7.608059306694137e-05, + "loss": 3.7757513382530306e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 774, + "tokens/total": 11685696, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 250222 + }, + { + "epoch": 12.109375, + "grad_norm": 0.00019213146879337728, + "learning_rate": 7.601638274345803e-05, + "loss": 4.42586360804853e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 775, + "tokens/total": 11700828, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 250546 + }, + { + "epoch": 12.125, + "grad_norm": 7.640149124199525e-05, + "learning_rate": 7.595211764629117e-05, + "loss": 2.4050909814832266e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 776, + "tokens/total": 11715968, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 250869 + }, + { + "epoch": 12.140625, + "grad_norm": 0.00014666939387097955, + "learning_rate": 7.588779794293086e-05, + "loss": 4.098519639228471e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 777, + "tokens/total": 11731076, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 251195 + }, + { + "epoch": 12.15625, + "grad_norm": 0.00011734114377759397, + "learning_rate": 7.582342380100953e-05, + "loss": 4.005758910352597e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 778, + "tokens/total": 11746184, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 251521 + }, + { + "epoch": 12.171875, + "grad_norm": 9.200117347063497e-05, + "learning_rate": 7.575899538830149e-05, + "loss": 3.269858098065015e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 779, + "tokens/total": 11761240, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 251840 + }, + { + "epoch": 12.1875, + "grad_norm": 6.822905561421067e-05, + "learning_rate": 7.569451287272247e-05, + "loss": 2.2363017251336714e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 780, + "tokens/total": 11776172, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 252166 + }, + { + "epoch": 12.203125, + "grad_norm": 0.00040962512139230967, + "learning_rate": 7.562997642232921e-05, + "loss": 4.020983851660276e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 781, + "tokens/total": 11791252, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 252493 + }, + { + "epoch": 12.21875, + "grad_norm": 6.947588553884998e-05, + "learning_rate": 7.556538620531904e-05, + "loss": 2.264953536723624e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 782, + "tokens/total": 11806280, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 252815 + }, + { + "epoch": 12.234375, + "grad_norm": 0.00028148273122496903, + "learning_rate": 7.55007423900294e-05, + "loss": 7.102900781319477e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 783, + "tokens/total": 11821296, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 253138 + }, + { + "epoch": 12.25, + "grad_norm": 0.0005345477839000523, + "learning_rate": 7.54360451449374e-05, + "loss": 4.053899829159491e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 784, + "tokens/total": 11836420, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 253459 + }, + { + "epoch": 12.265625, + "grad_norm": 0.00013953889720141888, + "learning_rate": 7.537129463865945e-05, + "loss": 4.29206011176575e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 785, + "tokens/total": 11851556, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 253785 + }, + { + "epoch": 12.28125, + "grad_norm": 0.0001949247089214623, + "learning_rate": 7.530649103995074e-05, + "loss": 3.856100192933809e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 786, + "tokens/total": 11866676, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 254111 + }, + { + "epoch": 12.296875, + "grad_norm": 0.0003590922278817743, + "learning_rate": 7.524163451770481e-05, + "loss": 1.007770697469823e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 787, + "tokens/total": 11881856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 254433 + }, + { + "epoch": 12.3125, + "grad_norm": 0.0001303488970734179, + "learning_rate": 7.517672524095319e-05, + "loss": 4.375161097414093e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 788, + "tokens/total": 11897048, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 254756 + }, + { + "epoch": 12.328125, + "grad_norm": 0.00010425105574540794, + "learning_rate": 7.511176337886485e-05, + "loss": 3.4424488148943055e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 789, + "tokens/total": 11912072, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 255074 + }, + { + "epoch": 12.34375, + "grad_norm": 0.00011002722021657974, + "learning_rate": 7.50467491007458e-05, + "loss": 3.731329115908011e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 790, + "tokens/total": 11927120, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 255399 + }, + { + "epoch": 12.359375, + "grad_norm": 8.662894833832979e-05, + "learning_rate": 7.498168257603875e-05, + "loss": 2.95067320621456e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 791, + "tokens/total": 11942172, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 255723 + }, + { + "epoch": 12.375, + "grad_norm": 0.00018941835151053965, + "learning_rate": 7.491656397432247e-05, + "loss": 4.5850742935726885e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 792, + "tokens/total": 11957288, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 256046 + }, + { + "epoch": 12.390625, + "grad_norm": 7.976038614287972e-05, + "learning_rate": 7.485139346531149e-05, + "loss": 2.5515669221931603e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 793, + "tokens/total": 11972424, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 256368 + }, + { + "epoch": 12.40625, + "grad_norm": 9.235714969690889e-05, + "learning_rate": 7.478617121885563e-05, + "loss": 2.5488184292044025e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 794, + "tokens/total": 11987508, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 256692 + }, + { + "epoch": 12.421875, + "grad_norm": 0.00018081159214489162, + "learning_rate": 7.472089740493959e-05, + "loss": 3.463078883214621e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 795, + "tokens/total": 12002564, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 257012 + }, + { + "epoch": 12.4375, + "grad_norm": 0.00010349749936722219, + "learning_rate": 7.46555721936824e-05, + "loss": 2.919844519055914e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 796, + "tokens/total": 12017704, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 257339 + }, + { + "epoch": 12.453125, + "grad_norm": 0.00013093859888613224, + "learning_rate": 7.459019575533707e-05, + "loss": 4.470029125513975e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 797, + "tokens/total": 12032856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 257664 + }, + { + "epoch": 12.46875, + "grad_norm": 0.00011427248682593927, + "learning_rate": 7.452476826029011e-05, + "loss": 3.6278854622651124e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 798, + "tokens/total": 12048028, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 257988 + }, + { + "epoch": 12.484375, + "grad_norm": 0.00018132558034267277, + "learning_rate": 7.445928987906114e-05, + "loss": 3.984095201303717e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 799, + "tokens/total": 12063092, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 258315 + }, + { + "epoch": 12.5, + "grad_norm": 0.0001274140231544152, + "learning_rate": 7.439376078230234e-05, + "loss": 3.1628230772184907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 800, + "tokens/total": 12078208, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 258638 + }, + { + "epoch": 12.515625, + "grad_norm": 0.00011142557195853442, + "learning_rate": 7.432818114079813e-05, + "loss": 2.8811889478674857e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 801, + "tokens/total": 12093348, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 258963 + }, + { + "epoch": 12.53125, + "grad_norm": 0.00019037636229768395, + "learning_rate": 7.426255112546458e-05, + "loss": 4.9189375204150565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 802, + "tokens/total": 12108412, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 259287 + }, + { + "epoch": 12.546875, + "grad_norm": 0.00019567548588383943, + "learning_rate": 7.419687090734912e-05, + "loss": 5.351162144506816e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 803, + "tokens/total": 12123480, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 259607 + }, + { + "epoch": 12.5625, + "grad_norm": 6.81504825479351e-05, + "learning_rate": 7.413114065762999e-05, + "loss": 2.2098488443589304e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 804, + "tokens/total": 12138644, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 259933 + }, + { + "epoch": 12.578125, + "grad_norm": 0.00010241116979159415, + "learning_rate": 7.406536054761584e-05, + "loss": 2.833344979080721e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 805, + "tokens/total": 12153664, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 260256 + }, + { + "epoch": 12.59375, + "grad_norm": 0.00022034579887986183, + "learning_rate": 7.399953074874523e-05, + "loss": 3.5401226341491565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 806, + "tokens/total": 12168872, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 260582 + }, + { + "epoch": 12.609375, + "grad_norm": 0.00010138696961803362, + "learning_rate": 7.393365143258626e-05, + "loss": 3.6219762478140183e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 807, + "tokens/total": 12184012, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 260909 + }, + { + "epoch": 12.625, + "grad_norm": 8.012948819668964e-05, + "learning_rate": 7.386772277083606e-05, + "loss": 2.6036345843749586e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 808, + "tokens/total": 12198968, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 261237 + }, + { + "epoch": 12.640625, + "grad_norm": 9.979843162000179e-05, + "learning_rate": 7.38017449353204e-05, + "loss": 3.229568392271176e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 809, + "tokens/total": 12214048, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 261560 + }, + { + "epoch": 12.65625, + "grad_norm": 9.405840683029965e-05, + "learning_rate": 7.373571809799319e-05, + "loss": 3.057072490264545e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 810, + "tokens/total": 12229144, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 261882 + }, + { + "epoch": 12.671875, + "grad_norm": 0.000499399786349386, + "learning_rate": 7.3669642430936e-05, + "loss": 1.0245048542856239e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 811, + "tokens/total": 12244368, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 262199 + }, + { + "epoch": 12.6875, + "grad_norm": 5.618187424261123e-05, + "learning_rate": 7.360351810635774e-05, + "loss": 1.7202623894263525e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 812, + "tokens/total": 12259492, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 262525 + }, + { + "epoch": 12.703125, + "grad_norm": 0.00023343712382484227, + "learning_rate": 7.353734529659409e-05, + "loss": 4.232795163261471e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 813, + "tokens/total": 12274608, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 262850 + }, + { + "epoch": 12.71875, + "grad_norm": 8.467691077385098e-05, + "learning_rate": 7.347112417410712e-05, + "loss": 2.407495458101039e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 814, + "tokens/total": 12289780, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 263171 + }, + { + "epoch": 12.734375, + "grad_norm": 0.0001598702510818839, + "learning_rate": 7.340485491148478e-05, + "loss": 3.024760644620983e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 815, + "tokens/total": 12304880, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 263495 + }, + { + "epoch": 12.75, + "grad_norm": 0.00011422912211855873, + "learning_rate": 7.33385376814405e-05, + "loss": 3.5377588574192487e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 816, + "tokens/total": 12319984, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 263818 + }, + { + "epoch": 12.765625, + "grad_norm": 4.749396612169221e-05, + "learning_rate": 7.327217265681276e-05, + "loss": 1.3899225450586528e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 817, + "tokens/total": 12335172, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 264139 + }, + { + "epoch": 12.78125, + "grad_norm": 9.546629007672891e-05, + "learning_rate": 7.320576001056452e-05, + "loss": 3.015875790879363e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 818, + "tokens/total": 12350312, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 264466 + }, + { + "epoch": 12.796875, + "grad_norm": 5.463911293190904e-05, + "learning_rate": 7.313929991578294e-05, + "loss": 1.6523881640750915e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 819, + "tokens/total": 12365496, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 264789 + }, + { + "epoch": 12.8125, + "grad_norm": 9.755798237165436e-05, + "learning_rate": 7.307279254567882e-05, + "loss": 2.3212492124002893e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 820, + "tokens/total": 12380560, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 265112 + }, + { + "epoch": 12.828125, + "grad_norm": 0.00010264299635309726, + "learning_rate": 7.300623807358614e-05, + "loss": 3.2085883958643535e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 821, + "tokens/total": 12395512, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 265435 + }, + { + "epoch": 12.84375, + "grad_norm": 8.120817801682279e-05, + "learning_rate": 7.293963667296164e-05, + "loss": 1.942413064170978e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 822, + "tokens/total": 12410528, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 265759 + }, + { + "epoch": 12.859375, + "grad_norm": 0.0003652231825981289, + "learning_rate": 7.287298851738442e-05, + "loss": 4.6293007471831515e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 823, + "tokens/total": 12425616, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 266081 + }, + { + "epoch": 12.875, + "grad_norm": 0.0002639541926328093, + "learning_rate": 7.280629378055536e-05, + "loss": 4.974856892658863e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 824, + "tokens/total": 12440816, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 266400 + }, + { + "epoch": 12.890625, + "grad_norm": 7.256776734720916e-05, + "learning_rate": 7.273955263629683e-05, + "loss": 2.3927705115056597e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 825, + "tokens/total": 12455896, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 266720 + }, + { + "epoch": 12.90625, + "grad_norm": 7.548509165644646e-05, + "learning_rate": 7.267276525855204e-05, + "loss": 2.0779016267624684e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 826, + "tokens/total": 12470980, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 267036 + }, + { + "epoch": 12.921875, + "grad_norm": 0.00012191422865726054, + "learning_rate": 7.260593182138478e-05, + "loss": 3.6637727589550195e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 827, + "tokens/total": 12486052, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 267361 + }, + { + "epoch": 12.9375, + "grad_norm": 0.0009720345260575414, + "learning_rate": 7.253905249897888e-05, + "loss": 1.0973613825626671e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 828, + "tokens/total": 12501208, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 267688 + }, + { + "epoch": 12.953125, + "grad_norm": 7.413590356009081e-05, + "learning_rate": 7.24721274656377e-05, + "loss": 2.2443480247602565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 829, + "tokens/total": 12516164, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 268009 + }, + { + "epoch": 12.96875, + "grad_norm": 0.00018752520554699004, + "learning_rate": 7.240515689578378e-05, + "loss": 4.03012245442369e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 830, + "tokens/total": 12531236, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 268332 + }, + { + "epoch": 12.984375, + "grad_norm": 7.950794679345563e-05, + "learning_rate": 7.233814096395834e-05, + "loss": 2.00104113901034e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 831, + "tokens/total": 12546304, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 268654 + }, + { + "epoch": 13.0, + "grad_norm": 0.0001989665033761412, + "learning_rate": 7.22710798448208e-05, + "loss": 5.7647207540867385e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 832, + "tokens/total": 12561532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 268970 + }, + { + "epoch": 13.015625, + "grad_norm": 0.00012065668124705553, + "learning_rate": 7.220397371314832e-05, + "loss": 3.097037961197202e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 833, + "tokens/total": 12576708, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 269295 + }, + { + "epoch": 13.03125, + "grad_norm": 0.0001326342608081177, + "learning_rate": 7.213682274383549e-05, + "loss": 3.3605601856834255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 834, + "tokens/total": 12591696, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 269614 + }, + { + "epoch": 13.046875, + "grad_norm": 0.00010373663099016994, + "learning_rate": 7.206962711189361e-05, + "loss": 2.7149353627464734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 835, + "tokens/total": 12606712, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 269937 + }, + { + "epoch": 13.0625, + "grad_norm": 9.629355918150395e-05, + "learning_rate": 7.20023869924505e-05, + "loss": 3.0645442166132852e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 836, + "tokens/total": 12621748, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 270258 + }, + { + "epoch": 13.078125, + "grad_norm": 4.5782438974129036e-05, + "learning_rate": 7.193510256074983e-05, + "loss": 1.4525633105222369e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 837, + "tokens/total": 12636812, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 270586 + }, + { + "epoch": 13.09375, + "grad_norm": 0.00029143388383090496, + "learning_rate": 7.186777399215087e-05, + "loss": 3.8549328564840835e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 838, + "tokens/total": 12651848, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 270908 + }, + { + "epoch": 13.109375, + "grad_norm": 6.666341505479068e-05, + "learning_rate": 7.18004014621278e-05, + "loss": 2.207491434091935e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 839, + "tokens/total": 12666920, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 271225 + }, + { + "epoch": 13.125, + "grad_norm": 6.579246837645769e-05, + "learning_rate": 7.173298514626945e-05, + "loss": 2.0363977455417626e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 840, + "tokens/total": 12682052, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 271551 + }, + { + "epoch": 13.140625, + "grad_norm": 0.00018273077148478478, + "learning_rate": 7.166552522027874e-05, + "loss": 2.352761612200993e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 841, + "tokens/total": 12697224, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 271879 + }, + { + "epoch": 13.15625, + "grad_norm": 5.646814315696247e-05, + "learning_rate": 7.159802185997228e-05, + "loss": 1.7437188262192649e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 842, + "tokens/total": 12712252, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 272200 + }, + { + "epoch": 13.171875, + "grad_norm": 0.00011839919170597568, + "learning_rate": 7.153047524127982e-05, + "loss": 3.6436388199945213e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 843, + "tokens/total": 12727476, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 272523 + }, + { + "epoch": 13.1875, + "grad_norm": 0.0001606152654858306, + "learning_rate": 7.146288554024389e-05, + "loss": 2.839481567207258e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 844, + "tokens/total": 12742488, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 272843 + }, + { + "epoch": 13.203125, + "grad_norm": 6.378227408276871e-05, + "learning_rate": 7.139525293301933e-05, + "loss": 2.080838839901844e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 845, + "tokens/total": 12757500, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 273165 + }, + { + "epoch": 13.21875, + "grad_norm": 0.0004620689433068037, + "learning_rate": 7.132757759587276e-05, + "loss": 2.3342386157310102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 846, + "tokens/total": 12772632, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 273489 + }, + { + "epoch": 13.234375, + "grad_norm": 0.00010379543527960777, + "learning_rate": 7.125985970518216e-05, + "loss": 2.6414465992274927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 847, + "tokens/total": 12787728, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 273814 + }, + { + "epoch": 13.25, + "grad_norm": 7.535841723438352e-05, + "learning_rate": 7.119209943743645e-05, + "loss": 2.522584281905438e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 848, + "tokens/total": 12802904, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 274141 + }, + { + "epoch": 13.265625, + "grad_norm": 7.253308285726234e-05, + "learning_rate": 7.112429696923496e-05, + "loss": 2.0930983737343922e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 849, + "tokens/total": 12818004, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 274467 + }, + { + "epoch": 13.28125, + "grad_norm": 0.00010928154370049015, + "learning_rate": 7.105645247728703e-05, + "loss": 2.7142471026309067e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 850, + "tokens/total": 12833000, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 274788 + }, + { + "epoch": 13.296875, + "grad_norm": 0.0001093312821467407, + "learning_rate": 7.098856613841153e-05, + "loss": 2.9678822102141567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 851, + "tokens/total": 12848060, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 275107 + }, + { + "epoch": 13.3125, + "grad_norm": 0.00017620428116060793, + "learning_rate": 7.092063812953636e-05, + "loss": 3.679098654174595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 852, + "tokens/total": 12863264, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 275429 + }, + { + "epoch": 13.328125, + "grad_norm": 6.490548548754305e-05, + "learning_rate": 7.085266862769804e-05, + "loss": 2.1255050342006143e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 853, + "tokens/total": 12878476, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 275754 + }, + { + "epoch": 13.34375, + "grad_norm": 0.0001361994945909828, + "learning_rate": 7.078465781004124e-05, + "loss": 3.864805421471829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 854, + "tokens/total": 12893452, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 276074 + }, + { + "epoch": 13.359375, + "grad_norm": 0.00023334445722866803, + "learning_rate": 7.071660585381828e-05, + "loss": 3.4866545775003033e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 855, + "tokens/total": 12908556, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 276397 + }, + { + "epoch": 13.375, + "grad_norm": 6.25511966063641e-05, + "learning_rate": 7.064851293638875e-05, + "loss": 1.7967286112252623e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 856, + "tokens/total": 12923656, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 276721 + }, + { + "epoch": 13.390625, + "grad_norm": 0.00011527373135322705, + "learning_rate": 7.05803792352189e-05, + "loss": 3.137032763333991e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 857, + "tokens/total": 12938808, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 277050 + }, + { + "epoch": 13.40625, + "grad_norm": 5.9854650316992775e-05, + "learning_rate": 7.05122049278814e-05, + "loss": 1.873960968623578e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 858, + "tokens/total": 12954044, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 277379 + }, + { + "epoch": 13.421875, + "grad_norm": 0.0001469842973165214, + "learning_rate": 7.044399019205467e-05, + "loss": 4.01368106395239e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 859, + "tokens/total": 12969096, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 277701 + }, + { + "epoch": 13.4375, + "grad_norm": 0.0001554029731778428, + "learning_rate": 7.037573520552248e-05, + "loss": 3.505352879074053e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 860, + "tokens/total": 12984184, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 278019 + }, + { + "epoch": 13.453125, + "grad_norm": 0.0006542939227074385, + "learning_rate": 7.030744014617355e-05, + "loss": 7.036339866317576e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 861, + "tokens/total": 12999240, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 278340 + }, + { + "epoch": 13.46875, + "grad_norm": 7.414568972308189e-05, + "learning_rate": 7.023910519200101e-05, + "loss": 1.9342680843692506e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 862, + "tokens/total": 13014336, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 278662 + }, + { + "epoch": 13.484375, + "grad_norm": 0.0002670721150934696, + "learning_rate": 7.017073052110199e-05, + "loss": 2.1539760837185895e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 863, + "tokens/total": 13029416, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 278986 + }, + { + "epoch": 13.5, + "grad_norm": 9.907476487569511e-05, + "learning_rate": 7.01023163116771e-05, + "loss": 2.038872025877936e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 864, + "tokens/total": 13044640, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 279311 + }, + { + "epoch": 13.515625, + "grad_norm": 4.2248771933373064e-05, + "learning_rate": 7.003386274203004e-05, + "loss": 1.1971330877713626e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 865, + "tokens/total": 13059684, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 279634 + }, + { + "epoch": 13.53125, + "grad_norm": 0.00012219577911309898, + "learning_rate": 6.996536999056702e-05, + "loss": 2.518343990232097e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 866, + "tokens/total": 13074760, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 279960 + }, + { + "epoch": 13.546875, + "grad_norm": 9.231813601218164e-05, + "learning_rate": 6.989683823579647e-05, + "loss": 3.010521140822675e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 867, + "tokens/total": 13089980, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 280284 + }, + { + "epoch": 13.5625, + "grad_norm": 0.00019922599312849343, + "learning_rate": 6.982826765632837e-05, + "loss": 2.023281240326469e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 868, + "tokens/total": 13104960, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 280607 + }, + { + "epoch": 13.578125, + "grad_norm": 8.861973037710413e-05, + "learning_rate": 6.975965843087396e-05, + "loss": 2.6947641345032025e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 869, + "tokens/total": 13119920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 280927 + }, + { + "epoch": 13.59375, + "grad_norm": 9.817941463552415e-05, + "learning_rate": 6.969101073824514e-05, + "loss": 2.44341049437935e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 870, + "tokens/total": 13135080, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 281246 + }, + { + "epoch": 13.609375, + "grad_norm": 9.399205737281591e-05, + "learning_rate": 6.962232475735414e-05, + "loss": 2.4531620965717593e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 871, + "tokens/total": 13150184, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 281569 + }, + { + "epoch": 13.625, + "grad_norm": 0.00012045784387737513, + "learning_rate": 6.95536006672129e-05, + "loss": 1.9861695363943e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 872, + "tokens/total": 13165276, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 281892 + }, + { + "epoch": 13.640625, + "grad_norm": 4.345045817899518e-05, + "learning_rate": 6.948483864693274e-05, + "loss": 1.347378884020145e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 873, + "tokens/total": 13180380, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 282209 + }, + { + "epoch": 13.65625, + "grad_norm": 5.4295047448249534e-05, + "learning_rate": 6.941603887572378e-05, + "loss": 1.7618372112337966e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 874, + "tokens/total": 13195420, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 282528 + }, + { + "epoch": 13.671875, + "grad_norm": 3.7880199670325965e-05, + "learning_rate": 6.93472015328946e-05, + "loss": 1.1165370779053774e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 875, + "tokens/total": 13210508, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 282852 + }, + { + "epoch": 13.6875, + "grad_norm": 6.62184611428529e-05, + "learning_rate": 6.927832679785162e-05, + "loss": 2.0546376617858186e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 876, + "tokens/total": 13225704, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 283177 + }, + { + "epoch": 13.703125, + "grad_norm": 0.00019075955788139254, + "learning_rate": 6.92094148500988e-05, + "loss": 3.2347554679290624e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 877, + "tokens/total": 13240808, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 283502 + }, + { + "epoch": 13.71875, + "grad_norm": 7.311074296012521e-05, + "learning_rate": 6.914046586923699e-05, + "loss": 2.3871341454650974e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 878, + "tokens/total": 13255932, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 283826 + }, + { + "epoch": 13.734375, + "grad_norm": 3.782408020924777e-05, + "learning_rate": 6.907148003496365e-05, + "loss": 1.0691402394513716e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 879, + "tokens/total": 13271004, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 284153 + }, + { + "epoch": 13.75, + "grad_norm": 7.222572457976639e-05, + "learning_rate": 6.900245752707218e-05, + "loss": 2.3125546704250155e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 880, + "tokens/total": 13286120, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 284479 + }, + { + "epoch": 13.765625, + "grad_norm": 5.55890474061016e-05, + "learning_rate": 6.893339852545169e-05, + "loss": 1.693718218120921e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 881, + "tokens/total": 13301260, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 284804 + }, + { + "epoch": 13.78125, + "grad_norm": 0.00011347887630108744, + "learning_rate": 6.886430321008632e-05, + "loss": 2.710476337597356e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 882, + "tokens/total": 13316384, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 285126 + }, + { + "epoch": 13.796875, + "grad_norm": 0.00036508546327240765, + "learning_rate": 6.879517176105485e-05, + "loss": 7.0446722020278685e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 883, + "tokens/total": 13331492, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 285449 + }, + { + "epoch": 13.8125, + "grad_norm": 0.00011320950579829514, + "learning_rate": 6.872600435853025e-05, + "loss": 3.2864747936400818e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 884, + "tokens/total": 13346580, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 285770 + }, + { + "epoch": 13.828125, + "grad_norm": 5.419834633357823e-05, + "learning_rate": 6.865680118277922e-05, + "loss": 1.5860000530665275e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 885, + "tokens/total": 13361648, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 286101 + }, + { + "epoch": 13.84375, + "grad_norm": 4.323993198340759e-05, + "learning_rate": 6.858756241416163e-05, + "loss": 1.1729822517736466e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 886, + "tokens/total": 13376720, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 286424 + }, + { + "epoch": 13.859375, + "grad_norm": 5.6182903790613636e-05, + "learning_rate": 6.85182882331302e-05, + "loss": 1.382684331474593e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 887, + "tokens/total": 13391656, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 286745 + }, + { + "epoch": 13.875, + "grad_norm": 5.661810064339079e-05, + "learning_rate": 6.844897882022984e-05, + "loss": 1.716691713227192e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 888, + "tokens/total": 13406776, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 287068 + }, + { + "epoch": 13.890625, + "grad_norm": 0.00019190310558769852, + "learning_rate": 6.837963435609737e-05, + "loss": 1.7268373539991444e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 889, + "tokens/total": 13421912, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 287394 + }, + { + "epoch": 13.90625, + "grad_norm": 5.752412835136056e-05, + "learning_rate": 6.831025502146093e-05, + "loss": 1.76233402271464e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 890, + "tokens/total": 13436940, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 287718 + }, + { + "epoch": 13.921875, + "grad_norm": 6.211781146703288e-05, + "learning_rate": 6.824084099713952e-05, + "loss": 1.4081907693253015e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 891, + "tokens/total": 13452080, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 288049 + }, + { + "epoch": 13.9375, + "grad_norm": 0.0005943211144767702, + "learning_rate": 6.817139246404257e-05, + "loss": 6.830468009866308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 892, + "tokens/total": 13467180, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 288368 + }, + { + "epoch": 13.953125, + "grad_norm": 5.17886619491037e-05, + "learning_rate": 6.810190960316948e-05, + "loss": 1.4416748399526114e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 893, + "tokens/total": 13482280, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 288691 + }, + { + "epoch": 13.96875, + "grad_norm": 4.485929821385071e-05, + "learning_rate": 6.803239259560903e-05, + "loss": 1.2776880566889304e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 894, + "tokens/total": 13497352, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 289015 + }, + { + "epoch": 13.984375, + "grad_norm": 5.71084747207351e-05, + "learning_rate": 6.796284162253911e-05, + "loss": 1.5132907265069662e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 895, + "tokens/total": 13512400, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 289336 + }, + { + "epoch": 14.0, + "grad_norm": 7.185615686466917e-05, + "learning_rate": 6.789325686522602e-05, + "loss": 1.277236151508987e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 896, + "tokens/total": 13527548, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 289660 + }, + { + "epoch": 14.015625, + "grad_norm": 4.978731885785237e-05, + "learning_rate": 6.782363850502417e-05, + "loss": 1.2179939403722528e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 897, + "tokens/total": 13542724, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 289984 + }, + { + "epoch": 14.03125, + "grad_norm": 8.392137533519417e-05, + "learning_rate": 6.775398672337556e-05, + "loss": 1.799379106159904e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 898, + "tokens/total": 13557752, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 290306 + }, + { + "epoch": 14.046875, + "grad_norm": 0.0001453920267522335, + "learning_rate": 6.768430170180924e-05, + "loss": 1.989572183447308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 899, + "tokens/total": 13572924, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 290629 + }, + { + "epoch": 14.0625, + "grad_norm": 5.724963193642907e-05, + "learning_rate": 6.761458362194097e-05, + "loss": 1.6024107480916427e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 900, + "tokens/total": 13588024, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 290959 + }, + { + "epoch": 14.078125, + "grad_norm": 8.20040368125774e-05, + "learning_rate": 6.754483266547253e-05, + "loss": 2.103687393173459e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 901, + "tokens/total": 13603244, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 291284 + }, + { + "epoch": 14.09375, + "grad_norm": 8.981539576780051e-05, + "learning_rate": 6.747504901419158e-05, + "loss": 2.5236784040316707e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 902, + "tokens/total": 13618464, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 291607 + }, + { + "epoch": 14.109375, + "grad_norm": 4.635561344912276e-05, + "learning_rate": 6.740523284997083e-05, + "loss": 1.211012204294093e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 903, + "tokens/total": 13633456, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 291932 + }, + { + "epoch": 14.125, + "grad_norm": 0.00024733011377975345, + "learning_rate": 6.733538435476779e-05, + "loss": 1.3879935067961924e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 904, + "tokens/total": 13648544, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 292255 + }, + { + "epoch": 14.140625, + "grad_norm": 0.00010022116475738585, + "learning_rate": 6.726550371062418e-05, + "loss": 1.9328476810187567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 905, + "tokens/total": 13663696, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 292580 + }, + { + "epoch": 14.15625, + "grad_norm": 9.24205596675165e-05, + "learning_rate": 6.719559109966563e-05, + "loss": 1.4655335007773829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 906, + "tokens/total": 13678704, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 292902 + }, + { + "epoch": 14.171875, + "grad_norm": 3.6749897844856605e-05, + "learning_rate": 6.712564670410095e-05, + "loss": 1.0849743148355628e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 907, + "tokens/total": 13693768, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 293226 + }, + { + "epoch": 14.1875, + "grad_norm": 4.9313708586851135e-05, + "learning_rate": 6.705567070622186e-05, + "loss": 1.3934755997979664e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 908, + "tokens/total": 13708908, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 293551 + }, + { + "epoch": 14.203125, + "grad_norm": 0.00010596215724945068, + "learning_rate": 6.698566328840242e-05, + "loss": 2.0530846995825414e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 909, + "tokens/total": 13724060, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 293875 + }, + { + "epoch": 14.21875, + "grad_norm": 6.745105929439887e-05, + "learning_rate": 6.69156246330986e-05, + "loss": 2.046339204753167e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 910, + "tokens/total": 13739192, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 294200 + }, + { + "epoch": 14.234375, + "grad_norm": 3.114332866971381e-05, + "learning_rate": 6.684555492284775e-05, + "loss": 7.437391218445555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 911, + "tokens/total": 13754336, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 294526 + }, + { + "epoch": 14.25, + "grad_norm": 8.269361569546163e-05, + "learning_rate": 6.677545434026816e-05, + "loss": 2.564119085946004e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 912, + "tokens/total": 13769412, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 294850 + }, + { + "epoch": 14.265625, + "grad_norm": 0.00011888529115822166, + "learning_rate": 6.670532306805861e-05, + "loss": 2.417497853457462e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 913, + "tokens/total": 13784544, + "tokens/train_per_sec_per_gpu": 8.61, + "tokens/trainable": 295174 + }, + { + "epoch": 14.28125, + "grad_norm": 6.309825403150171e-05, + "learning_rate": 6.663516128899785e-05, + "loss": 1.8808964341587853e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 914, + "tokens/total": 13799696, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 295496 + }, + { + "epoch": 14.296875, + "grad_norm": 5.9262347349431366e-05, + "learning_rate": 6.656496918594413e-05, + "loss": 1.7357599517708877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 915, + "tokens/total": 13814812, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 295825 + }, + { + "epoch": 14.3125, + "grad_norm": 5.164822141523473e-05, + "learning_rate": 6.649474694183474e-05, + "loss": 1.2478254802772426e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 916, + "tokens/total": 13829944, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 296147 + }, + { + "epoch": 14.328125, + "grad_norm": 6.594896694878116e-05, + "learning_rate": 6.642449473968551e-05, + "loss": 1.6502298194609466e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 917, + "tokens/total": 13845084, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 296470 + }, + { + "epoch": 14.34375, + "grad_norm": 5.508278263732791e-05, + "learning_rate": 6.635421276259038e-05, + "loss": 1.3780652352579636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 918, + "tokens/total": 13860172, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 296797 + }, + { + "epoch": 14.359375, + "grad_norm": 5.368686106521636e-05, + "learning_rate": 6.628390119372085e-05, + "loss": 1.5078080650710035e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 919, + "tokens/total": 13875296, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 297118 + }, + { + "epoch": 14.375, + "grad_norm": 4.166882354184054e-05, + "learning_rate": 6.621356021632556e-05, + "loss": 1.2374182460916927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 920, + "tokens/total": 13890396, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 297441 + }, + { + "epoch": 14.390625, + "grad_norm": 4.058206468471326e-05, + "learning_rate": 6.614319001372981e-05, + "loss": 1.086644488168531e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 921, + "tokens/total": 13905476, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 297763 + }, + { + "epoch": 14.40625, + "grad_norm": 4.9323389248456806e-05, + "learning_rate": 6.607279076933509e-05, + "loss": 1.4575615523426677e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 922, + "tokens/total": 13920536, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 298088 + }, + { + "epoch": 14.421875, + "grad_norm": 3.589461266528815e-05, + "learning_rate": 6.600236266661848e-05, + "loss": 8.809178098090342e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 923, + "tokens/total": 13935580, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 298411 + }, + { + "epoch": 14.4375, + "grad_norm": 8.650182280689478e-05, + "learning_rate": 6.593190588913239e-05, + "loss": 1.5303246527764713e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 924, + "tokens/total": 13950700, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 298733 + }, + { + "epoch": 14.453125, + "grad_norm": 8.58692146721296e-05, + "learning_rate": 6.586142062050387e-05, + "loss": 2.0843222046096344e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 925, + "tokens/total": 13965820, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 299054 + }, + { + "epoch": 14.46875, + "grad_norm": 6.944965571165085e-05, + "learning_rate": 6.579090704443432e-05, + "loss": 1.1496265415189555e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 926, + "tokens/total": 13980948, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 299375 + }, + { + "epoch": 14.484375, + "grad_norm": 5.2342576964292675e-05, + "learning_rate": 6.572036534469879e-05, + "loss": 1.2702308822554187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 927, + "tokens/total": 13996084, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 299700 + }, + { + "epoch": 14.5, + "grad_norm": 5.443696500151418e-05, + "learning_rate": 6.564979570514577e-05, + "loss": 1.7568534076417563e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 928, + "tokens/total": 14011092, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 300022 + }, + { + "epoch": 14.515625, + "grad_norm": 0.00025401683524250984, + "learning_rate": 6.557919830969644e-05, + "loss": 4.309782980271848e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 929, + "tokens/total": 14026324, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 300348 + }, + { + "epoch": 14.53125, + "grad_norm": 7.833947893232107e-05, + "learning_rate": 6.550857334234442e-05, + "loss": 1.8433300965625676e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 930, + "tokens/total": 14041448, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 300671 + }, + { + "epoch": 14.546875, + "grad_norm": 8.168330532498658e-05, + "learning_rate": 6.543792098715513e-05, + "loss": 1.8892920934376889e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 931, + "tokens/total": 14056636, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 300994 + }, + { + "epoch": 14.5625, + "grad_norm": 5.483560016728006e-05, + "learning_rate": 6.536724142826536e-05, + "loss": 1.629071448405739e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 932, + "tokens/total": 14071748, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 301323 + }, + { + "epoch": 14.578125, + "grad_norm": 4.461768548935652e-05, + "learning_rate": 6.529653484988287e-05, + "loss": 1.0997707704518689e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 933, + "tokens/total": 14086812, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 301649 + }, + { + "epoch": 14.59375, + "grad_norm": 0.0001430799311492592, + "learning_rate": 6.522580143628573e-05, + "loss": 2.7947155558649683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 934, + "tokens/total": 14101904, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 301975 + }, + { + "epoch": 14.609375, + "grad_norm": 8.027078001759946e-05, + "learning_rate": 6.515504137182206e-05, + "loss": 1.9248398075433215e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 935, + "tokens/total": 14117012, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 302295 + }, + { + "epoch": 14.625, + "grad_norm": 9.045998012879863e-05, + "learning_rate": 6.508425484090935e-05, + "loss": 2.628022912176675e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 936, + "tokens/total": 14131976, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 302617 + }, + { + "epoch": 14.640625, + "grad_norm": 4.7691853978903964e-05, + "learning_rate": 6.501344202803414e-05, + "loss": 1.1184176855749683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 937, + "tokens/total": 14147008, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 302937 + }, + { + "epoch": 14.65625, + "grad_norm": 8.31821234896779e-05, + "learning_rate": 6.494260311775145e-05, + "loss": 2.248458258691244e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 938, + "tokens/total": 14162152, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 303265 + }, + { + "epoch": 14.671875, + "grad_norm": 7.890448614489287e-05, + "learning_rate": 6.487173829468424e-05, + "loss": 1.7639106317801634e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 939, + "tokens/total": 14177152, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 303589 + }, + { + "epoch": 14.6875, + "grad_norm": 0.0007585128187201917, + "learning_rate": 6.480084774352312e-05, + "loss": 9.449095159652643e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 940, + "tokens/total": 14192124, + "tokens/train_per_sec_per_gpu": 8.68, + "tokens/trainable": 303911 + }, + { + "epoch": 14.703125, + "grad_norm": 4.063158485223539e-05, + "learning_rate": 6.472993164902566e-05, + "loss": 1.0883406957873376e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 941, + "tokens/total": 14207176, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 304235 + }, + { + "epoch": 14.71875, + "grad_norm": 0.0012726637069135904, + "learning_rate": 6.465899019601607e-05, + "loss": 1.647085127842729e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 942, + "tokens/total": 14222308, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 304559 + }, + { + "epoch": 14.734375, + "grad_norm": 6.790161569369957e-05, + "learning_rate": 6.458802356938462e-05, + "loss": 2.0932579900545534e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 943, + "tokens/total": 14237520, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 304880 + }, + { + "epoch": 14.75, + "grad_norm": 5.292199057294056e-05, + "learning_rate": 6.451703195408714e-05, + "loss": 1.5438358786923345e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 944, + "tokens/total": 14252592, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 305200 + }, + { + "epoch": 14.765625, + "grad_norm": 6.903067696839571e-05, + "learning_rate": 6.444601553514472e-05, + "loss": 1.4309335938378354e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 945, + "tokens/total": 14267684, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 305522 + }, + { + "epoch": 14.78125, + "grad_norm": 6.407395994756371e-05, + "learning_rate": 6.437497449764292e-05, + "loss": 1.7573401009940426e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 946, + "tokens/total": 14282812, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 305844 + }, + { + "epoch": 14.796875, + "grad_norm": 7.578920485684648e-05, + "learning_rate": 6.43039090267316e-05, + "loss": 1.912631887535099e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 947, + "tokens/total": 14298036, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 306167 + }, + { + "epoch": 14.8125, + "grad_norm": 6.992624548729509e-05, + "learning_rate": 6.423281930762426e-05, + "loss": 1.189959675684804e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 948, + "tokens/total": 14313140, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 306489 + }, + { + "epoch": 14.828125, + "grad_norm": 6.511853280244395e-05, + "learning_rate": 6.416170552559758e-05, + "loss": 1.474600708206708e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 949, + "tokens/total": 14328408, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 306810 + }, + { + "epoch": 14.84375, + "grad_norm": 0.00010607110743876547, + "learning_rate": 6.409056786599093e-05, + "loss": 2.453754404996289e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 950, + "tokens/total": 14343448, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 307130 + }, + { + "epoch": 14.859375, + "grad_norm": 0.0001000507763819769, + "learning_rate": 6.401940651420598e-05, + "loss": 2.529926860006526e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 951, + "tokens/total": 14358560, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 307453 + }, + { + "epoch": 14.875, + "grad_norm": 0.00011187302879989147, + "learning_rate": 6.39482216557061e-05, + "loss": 2.2469866962637752e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 952, + "tokens/total": 14373584, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 307776 + }, + { + "epoch": 14.890625, + "grad_norm": 4.595956488628872e-05, + "learning_rate": 6.387701347601593e-05, + "loss": 1.499669224358513e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 953, + "tokens/total": 14388648, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 308096 + }, + { + "epoch": 14.90625, + "grad_norm": 6.818763358751312e-05, + "learning_rate": 6.380578216072088e-05, + "loss": 8.027869853322045e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 954, + "tokens/total": 14403620, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 308416 + }, + { + "epoch": 14.921875, + "grad_norm": 0.0001449510018574074, + "learning_rate": 6.373452789546669e-05, + "loss": 2.331412360945251e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 955, + "tokens/total": 14418724, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 308740 + }, + { + "epoch": 14.9375, + "grad_norm": 3.8131242035888135e-05, + "learning_rate": 6.36632508659589e-05, + "loss": 9.072298894352571e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 956, + "tokens/total": 14433808, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 309062 + }, + { + "epoch": 14.953125, + "grad_norm": 2.594317265902646e-05, + "learning_rate": 6.359195125796233e-05, + "loss": 6.713360676258162e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 957, + "tokens/total": 14448956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 309391 + }, + { + "epoch": 14.96875, + "grad_norm": 0.00010180116805713624, + "learning_rate": 6.352062925730074e-05, + "loss": 2.0256127299944637e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 958, + "tokens/total": 14463960, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 309716 + }, + { + "epoch": 14.984375, + "grad_norm": 3.978185850428417e-05, + "learning_rate": 6.344928504985616e-05, + "loss": 1.1063133342759102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 959, + "tokens/total": 14479012, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 310029 + }, + { + "epoch": 15.0, + "grad_norm": 4.233407889842056e-05, + "learning_rate": 6.337791882156857e-05, + "loss": 1.1700456070684595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 960, + "tokens/total": 14494116, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 310350 + }, + { + "epoch": 15.015625, + "grad_norm": 6.0397691413527355e-05, + "learning_rate": 6.330653075843527e-05, + "loss": 1.8204215166406357e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 961, + "tokens/total": 14509288, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 310676 + }, + { + "epoch": 15.03125, + "grad_norm": 4.4577634980669245e-05, + "learning_rate": 6.323512104651052e-05, + "loss": 1.2006689757981803e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 962, + "tokens/total": 14524308, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 310999 + }, + { + "epoch": 15.046875, + "grad_norm": 4.363745028967969e-05, + "learning_rate": 6.316368987190498e-05, + "loss": 1.057101371770841e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 963, + "tokens/total": 14539320, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 311326 + }, + { + "epoch": 15.0625, + "grad_norm": 4.965631160303019e-05, + "learning_rate": 6.309223742078527e-05, + "loss": 1.4942244206395117e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 964, + "tokens/total": 14554348, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 311652 + }, + { + "epoch": 15.078125, + "grad_norm": 4.124893530388363e-05, + "learning_rate": 6.302076387937343e-05, + "loss": 1.0145915894099744e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 965, + "tokens/total": 14569396, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 311979 + }, + { + "epoch": 15.09375, + "grad_norm": 0.00016004100325517356, + "learning_rate": 6.294926943394646e-05, + "loss": 2.7917315037484514e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 966, + "tokens/total": 14584500, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 312300 + }, + { + "epoch": 15.109375, + "grad_norm": 5.61064480280038e-05, + "learning_rate": 6.28777542708359e-05, + "loss": 1.3850698223905056e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 967, + "tokens/total": 14599592, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 312620 + }, + { + "epoch": 15.125, + "grad_norm": 6.103901250753552e-05, + "learning_rate": 6.280621857642722e-05, + "loss": 1.4942685311325477e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 968, + "tokens/total": 14614736, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 312944 + }, + { + "epoch": 15.140625, + "grad_norm": 2.391202906437684e-05, + "learning_rate": 6.273466253715941e-05, + "loss": 6.849907094874652e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 969, + "tokens/total": 14629748, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 313271 + }, + { + "epoch": 15.15625, + "grad_norm": 4.6358443796634674e-05, + "learning_rate": 6.266308633952454e-05, + "loss": 1.3744077023147838e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 970, + "tokens/total": 14644736, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 313592 + }, + { + "epoch": 15.171875, + "grad_norm": 3.74452029063832e-05, + "learning_rate": 6.259149017006713e-05, + "loss": 8.902447348191345e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 971, + "tokens/total": 14659816, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 313918 + }, + { + "epoch": 15.1875, + "grad_norm": 9.493020479567349e-05, + "learning_rate": 6.251987421538383e-05, + "loss": 1.6062704162322916e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 972, + "tokens/total": 14674920, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 314244 + }, + { + "epoch": 15.203125, + "grad_norm": 2.5273504434153438e-05, + "learning_rate": 6.244823866212282e-05, + "loss": 7.111624995559396e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 973, + "tokens/total": 14690064, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 314568 + }, + { + "epoch": 15.21875, + "grad_norm": 0.00010362095781601965, + "learning_rate": 6.237658369698335e-05, + "loss": 1.7851814391178777e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 974, + "tokens/total": 14705236, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 314888 + }, + { + "epoch": 15.234375, + "grad_norm": 6.892029341543093e-05, + "learning_rate": 6.23049095067153e-05, + "loss": 8.738239785088808e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 975, + "tokens/total": 14720384, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 315217 + }, + { + "epoch": 15.25, + "grad_norm": 0.00016985680849757046, + "learning_rate": 6.223321627811857e-05, + "loss": 1.4400820873561315e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 976, + "tokens/total": 14735432, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 315538 + }, + { + "epoch": 15.265625, + "grad_norm": 4.9234015023102984e-05, + "learning_rate": 6.216150419804279e-05, + "loss": 1.3229498563305242e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 977, + "tokens/total": 14750520, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 315862 + }, + { + "epoch": 15.28125, + "grad_norm": 6.438756099669263e-05, + "learning_rate": 6.208977345338665e-05, + "loss": 1.5429411632794654e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 978, + "tokens/total": 14765648, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 316185 + }, + { + "epoch": 15.296875, + "grad_norm": 3.929212471120991e-05, + "learning_rate": 6.201802423109749e-05, + "loss": 1.100590338865004e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 979, + "tokens/total": 14780724, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 316510 + }, + { + "epoch": 15.3125, + "grad_norm": 6.362700514728203e-05, + "learning_rate": 6.194625671817084e-05, + "loss": 1.5663575823054998e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 980, + "tokens/total": 14795824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 316830 + }, + { + "epoch": 15.328125, + "grad_norm": 5.856006464455277e-05, + "learning_rate": 6.187447110164986e-05, + "loss": 9.909776963468175e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 981, + "tokens/total": 14810876, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 317153 + }, + { + "epoch": 15.34375, + "grad_norm": 4.1728420910658315e-05, + "learning_rate": 6.180266756862493e-05, + "loss": 9.539714938000543e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 982, + "tokens/total": 14825992, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 317473 + }, + { + "epoch": 15.359375, + "grad_norm": 3.521126927807927e-05, + "learning_rate": 6.173084630623311e-05, + "loss": 9.559518048263271e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 983, + "tokens/total": 14840956, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 317796 + }, + { + "epoch": 15.375, + "grad_norm": 2.665440842974931e-05, + "learning_rate": 6.165900750165763e-05, + "loss": 6.843700930403429e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 984, + "tokens/total": 14856184, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 318123 + }, + { + "epoch": 15.390625, + "grad_norm": 7.02376346453093e-05, + "learning_rate": 6.15871513421275e-05, + "loss": 1.4553772871295223e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 985, + "tokens/total": 14871248, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 318449 + }, + { + "epoch": 15.40625, + "grad_norm": 8.755866292631254e-05, + "learning_rate": 6.151527801491692e-05, + "loss": 1.3671403849002672e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 986, + "tokens/total": 14886400, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 318775 + }, + { + "epoch": 15.421875, + "grad_norm": 6.377039971994236e-05, + "learning_rate": 6.144338770734487e-05, + "loss": 1.655393248256587e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 987, + "tokens/total": 14901536, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 319093 + }, + { + "epoch": 15.4375, + "grad_norm": 5.989666533423588e-05, + "learning_rate": 6.137148060677453e-05, + "loss": 9.670616236689966e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 988, + "tokens/total": 14916616, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 319418 + }, + { + "epoch": 15.453125, + "grad_norm": 5.5695742048555985e-05, + "learning_rate": 6.129955690061289e-05, + "loss": 8.72893622272386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 989, + "tokens/total": 14931756, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 319739 + }, + { + "epoch": 15.46875, + "grad_norm": 3.395605745026842e-05, + "learning_rate": 6.122761677631022e-05, + "loss": 7.767515057821583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 990, + "tokens/total": 14946856, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 320062 + }, + { + "epoch": 15.484375, + "grad_norm": 0.00013859746104571968, + "learning_rate": 6.115566042135953e-05, + "loss": 2.581086846475955e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 991, + "tokens/total": 14962100, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 320385 + }, + { + "epoch": 15.5, + "grad_norm": 3.372377614141442e-05, + "learning_rate": 6.10836880232962e-05, + "loss": 8.867686460689583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 992, + "tokens/total": 14977184, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 320706 + }, + { + "epoch": 15.515625, + "grad_norm": 6.984758510952815e-05, + "learning_rate": 6.1011699769697384e-05, + "loss": 1.2891443930129753e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 993, + "tokens/total": 14992300, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 321031 + }, + { + "epoch": 15.53125, + "grad_norm": 5.5518965382361785e-05, + "learning_rate": 6.0939695848181546e-05, + "loss": 1.398159497512097e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 994, + "tokens/total": 15007460, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 321349 + }, + { + "epoch": 15.546875, + "grad_norm": 8.458229422103614e-05, + "learning_rate": 6.086767644640801e-05, + "loss": 1.4708990647704923e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 995, + "tokens/total": 15022532, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 321672 + }, + { + "epoch": 15.5625, + "grad_norm": 0.00017973266949411482, + "learning_rate": 6.079564175207645e-05, + "loss": 3.492345513222972e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 996, + "tokens/total": 15037596, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 321995 + }, + { + "epoch": 15.578125, + "grad_norm": 5.182886525290087e-05, + "learning_rate": 6.072359195292635e-05, + "loss": 1.2431519280653447e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 997, + "tokens/total": 15052708, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 322320 + }, + { + "epoch": 15.59375, + "grad_norm": 0.00039384455885738134, + "learning_rate": 6.0651527236736636e-05, + "loss": 4.331072432250949e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 998, + "tokens/total": 15067820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 322642 + }, + { + "epoch": 15.609375, + "grad_norm": 6.89527005306445e-05, + "learning_rate": 6.057944779132505e-05, + "loss": 1.9015310499526095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 999, + "tokens/total": 15082956, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 322968 + }, + { + "epoch": 15.625, + "grad_norm": 0.00038482208037748933, + "learning_rate": 6.050735380454774e-05, + "loss": 4.186047590337694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1000, + "tokens/total": 15098020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 323293 + }, + { + "epoch": 15.640625, + "grad_norm": 2.5691933842608705e-05, + "learning_rate": 6.043524546429874e-05, + "loss": 7.110375008778647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1001, + "tokens/total": 15113016, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 323615 + }, + { + "epoch": 15.65625, + "grad_norm": 5.0753205869114026e-05, + "learning_rate": 6.036312295850954e-05, + "loss": 1.0369699339207727e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1002, + "tokens/total": 15128136, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 323945 + }, + { + "epoch": 15.671875, + "grad_norm": 1.725030415400397e-05, + "learning_rate": 6.029098647514848e-05, + "loss": 4.307669883019116e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1003, + "tokens/total": 15143328, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 324273 + }, + { + "epoch": 15.6875, + "grad_norm": 5.06673677591607e-05, + "learning_rate": 6.0218836202220394e-05, + "loss": 1.4467525488726096e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1004, + "tokens/total": 15158384, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 324598 + }, + { + "epoch": 15.703125, + "grad_norm": 2.792146915453486e-05, + "learning_rate": 6.014667232776601e-05, + "loss": 7.40162306556158e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1005, + "tokens/total": 15173492, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 324921 + }, + { + "epoch": 15.71875, + "grad_norm": 3.8122736441437155e-05, + "learning_rate": 6.0074495039861515e-05, + "loss": 1.123777792599867e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1006, + "tokens/total": 15188732, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 325244 + }, + { + "epoch": 15.734375, + "grad_norm": 8.14247687230818e-05, + "learning_rate": 6.000230452661806e-05, + "loss": 1.9589356270444114e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1007, + "tokens/total": 15203780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 325567 + }, + { + "epoch": 15.75, + "grad_norm": 5.886691724299453e-05, + "learning_rate": 5.993010097618127e-05, + "loss": 1.378559204567864e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1008, + "tokens/total": 15218936, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 325892 + }, + { + "epoch": 15.765625, + "grad_norm": 0.00011032194015569985, + "learning_rate": 5.985788457673074e-05, + "loss": 1.073725343303522e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1009, + "tokens/total": 15234040, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 326212 + }, + { + "epoch": 15.78125, + "grad_norm": 0.00013100061914883554, + "learning_rate": 5.9785655516479535e-05, + "loss": 2.953038574560196e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1010, + "tokens/total": 15249068, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 326535 + }, + { + "epoch": 15.796875, + "grad_norm": 4.720654396805912e-05, + "learning_rate": 5.9713413983673774e-05, + "loss": 7.844298011150386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1011, + "tokens/total": 15264252, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 326858 + }, + { + "epoch": 15.8125, + "grad_norm": 3.831119465758093e-05, + "learning_rate": 5.9641160166592026e-05, + "loss": 1.0103673275807523e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1012, + "tokens/total": 15279436, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 327178 + }, + { + "epoch": 15.828125, + "grad_norm": 4.244214505888522e-05, + "learning_rate": 5.9568894253544874e-05, + "loss": 1.1848846952489112e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1013, + "tokens/total": 15294388, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 327502 + }, + { + "epoch": 15.84375, + "grad_norm": 4.1179850086336955e-05, + "learning_rate": 5.9496616432874466e-05, + "loss": 1.0905631597779575e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1014, + "tokens/total": 15309404, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 327821 + }, + { + "epoch": 15.859375, + "grad_norm": 4.633567368728109e-05, + "learning_rate": 5.9424326892953975e-05, + "loss": 1.24127450362721e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1015, + "tokens/total": 15324568, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 328143 + }, + { + "epoch": 15.875, + "grad_norm": 3.0212768251658417e-05, + "learning_rate": 5.935202582218707e-05, + "loss": 8.720062965039688e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1016, + "tokens/total": 15339560, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 328462 + }, + { + "epoch": 15.890625, + "grad_norm": 7.697457476751879e-05, + "learning_rate": 5.9279713409007555e-05, + "loss": 1.3978632296129945e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1017, + "tokens/total": 15354696, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 328785 + }, + { + "epoch": 15.90625, + "grad_norm": 3.8135396607685834e-05, + "learning_rate": 5.92073898418787e-05, + "loss": 1.062007299879042e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1018, + "tokens/total": 15369816, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 329108 + }, + { + "epoch": 15.921875, + "grad_norm": 3.6240795452613384e-05, + "learning_rate": 5.9135055309292955e-05, + "loss": 9.362717605654325e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1019, + "tokens/total": 15384936, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 329429 + }, + { + "epoch": 15.9375, + "grad_norm": 3.131631819996983e-05, + "learning_rate": 5.906270999977121e-05, + "loss": 8.221573466471455e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1020, + "tokens/total": 15400068, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 329751 + }, + { + "epoch": 15.953125, + "grad_norm": 0.00010301479778718203, + "learning_rate": 5.899035410186259e-05, + "loss": 1.3009018857701449e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1021, + "tokens/total": 15415120, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 330075 + }, + { + "epoch": 15.96875, + "grad_norm": 3.9301598008023575e-05, + "learning_rate": 5.891798780414369e-05, + "loss": 1.0615477776809712e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1022, + "tokens/total": 15430256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 330394 + }, + { + "epoch": 15.984375, + "grad_norm": 9.136048902291805e-05, + "learning_rate": 5.884561129521832e-05, + "loss": 8.340676345142128e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1023, + "tokens/total": 15445400, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 330717 + }, + { + "epoch": 16.0, + "grad_norm": 4.0084210922941566e-05, + "learning_rate": 5.8773224763716795e-05, + "loss": 7.630780487488664e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1024, + "tokens/total": 15460436, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 331040 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0609825857985117e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-1024/training_args.bin b/aft/coin/checkpoint-1024/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-1024/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-128/README.md b/aft/coin/checkpoint-128/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-128/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-128/adapter_config.json b/aft/coin/checkpoint-128/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-128/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-128/adapter_model.safetensors b/aft/coin/checkpoint-128/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74ecabdc517f8b578a48affe47272c539c43ded1 --- /dev/null +++ b/aft/coin/checkpoint-128/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36e2edfa62a7cbc0fd2979952a28f749ccb218b930012a5af3d3081c8d08db9a +size 1047628488 diff --git a/aft/coin/checkpoint-128/chat_template.jinja b/aft/coin/checkpoint-128/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-128/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-128/tokenizer.json b/aft/coin/checkpoint-128/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-128/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-128/tokenizer_config.json b/aft/coin/checkpoint-128/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-128/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-128/tokens_state.json b/aft/coin/checkpoint-128/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..196f86423a0671ab5327df098b12ca4949e62a51 --- /dev/null +++ b/aft/coin/checkpoint-128/tokens_state.json @@ -0,0 +1 @@ +{"total": 1932952, "trainable": 41380} \ No newline at end of file diff --git a/aft/coin/checkpoint-128/trainer_state.json b/aft/coin/checkpoint-128/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8e4e319a77d8066f6ae15dfb14ede0abb7d82d08 --- /dev/null +++ b/aft/coin/checkpoint-128/trainer_state.json @@ -0,0 +1,1826 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 128, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0028458833694458, + "learning_rate": 6.274509803921569e-05, + "loss": 0.01862957514822483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0188, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.9001771807670593, + "learning_rate": 6.372549019607843e-05, + "loss": 0.021384725347161293, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02162, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.5960713028907776, + "learning_rate": 6.470588235294118e-05, + "loss": 0.010063373483717442, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01011, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 2.038512706756592, + "learning_rate": 6.568627450980392e-05, + "loss": 0.03831503912806511, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03906, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 2.5726208686828613, + "learning_rate": 6.666666666666667e-05, + "loss": 0.03134162724018097, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03184, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.793451189994812, + "learning_rate": 6.764705882352942e-05, + "loss": 0.028099995106458664, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0285, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 0.7262976169586182, + "learning_rate": 6.862745098039216e-05, + "loss": 0.02038084715604782, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02059, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 0.532096803188324, + "learning_rate": 6.96078431372549e-05, + "loss": 0.013697818852961063, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01379, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 0.7747638821601868, + "learning_rate": 7.058823529411765e-05, + "loss": 0.023866616189479828, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02415, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 0.49953746795654297, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014121961779892445, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01422, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.3340316116809845, + "learning_rate": 7.254901960784314e-05, + "loss": 0.012052969075739384, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01213, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.735864520072937, + "learning_rate": 7.352941176470589e-05, + "loss": 0.04014549404382706, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04096, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.4149185419082642, + "learning_rate": 7.450980392156864e-05, + "loss": 0.045495182275772095, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04655, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 0.31831473112106323, + "learning_rate": 7.549019607843137e-05, + "loss": 0.011568848043680191, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01164, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 0.5228955745697021, + "learning_rate": 7.647058823529411e-05, + "loss": 0.02045898698270321, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02067, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.5053377151489258, + "learning_rate": 7.745098039215687e-05, + "loss": 0.020432129502296448, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02064, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.35163524746894836, + "learning_rate": 7.843137254901961e-05, + "loss": 0.01403958909213543, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01414, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.6905986070632935, + "learning_rate": 7.941176470588235e-05, + "loss": 0.03329383581876755, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.03385, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 0.4459448456764221, + "learning_rate": 8.039215686274511e-05, + "loss": 0.006232030689716339, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00625, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.2638562321662903, + "learning_rate": 8.137254901960785e-05, + "loss": 0.006893642246723175, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00692, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 1.170509696006775, + "learning_rate": 8.23529411764706e-05, + "loss": 0.0398201048374176, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.04062, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.8102952837944031, + "learning_rate": 8.333333333333334e-05, + "loss": 0.018316198140382767, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01848, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.6812735795974731, + "learning_rate": 8.431372549019608e-05, + "loss": 0.02392837405204773, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02422, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.251705288887024, + "learning_rate": 8.529411764705883e-05, + "loss": 0.024196332320570946, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02449, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.5772960782051086, + "learning_rate": 8.627450980392158e-05, + "loss": 0.021560901775956154, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0218, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.6470917463302612, + "learning_rate": 8.725490196078432e-05, + "loss": 0.04491877555847168, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.04594, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.41440942883491516, + "learning_rate": 8.823529411764706e-05, + "loss": 0.0230612400919199, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02333, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 0.39720970392227173, + "learning_rate": 8.921568627450981e-05, + "loss": 0.013313441537320614, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0134, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.8898259401321411, + "learning_rate": 9.019607843137255e-05, + "loss": 0.015421045944094658, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01554, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 8.74992847442627, + "learning_rate": 9.11764705882353e-05, + "loss": 0.03271842002868652, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03326, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.18793171644210815, + "learning_rate": 9.215686274509804e-05, + "loss": 0.0015747153665870428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00158, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.9997888207435608, + "learning_rate": 9.313725490196079e-05, + "loss": 0.03334064409136772, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0339, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.6498851776123047, + "learning_rate": 9.411764705882353e-05, + "loss": 0.023218588903546333, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02349, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.5593264102935791, + "learning_rate": 9.509803921568627e-05, + "loss": 0.005530651658773422, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00555, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.8789476752281189, + "learning_rate": 9.607843137254903e-05, + "loss": 0.04554657265543938, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0466, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.5140370726585388, + "learning_rate": 9.705882352941177e-05, + "loss": 0.014101475477218628, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0142, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 0.5285096764564514, + "learning_rate": 9.80392156862745e-05, + "loss": 0.030611136928200722, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03108, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 0.6092981696128845, + "learning_rate": 9.901960784313727e-05, + "loss": 0.028998689725995064, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02942, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0235849618911743, + "learning_rate": 0.0001, + "loss": 0.025761116296052933, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0261, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.9383920431137085, + "learning_rate": 9.999994135965763e-05, + "loss": 0.02419213019311428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02449, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.8995917439460754, + "learning_rate": 9.999976543878332e-05, + "loss": 0.011181461624801159, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01124, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 3.9120519161224365, + "learning_rate": 9.999947223783559e-05, + "loss": 0.10826049745082855, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.11434, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 1.801079511642456, + "learning_rate": 9.999906175757857e-05, + "loss": 0.029263045638799667, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0297, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 3.040914535522461, + "learning_rate": 9.99985339990821e-05, + "loss": 0.04611361399292946, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04719, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.6034630537033081, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0116285290569067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0117, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.238577961921692, + "learning_rate": 9.999712665317819e-05, + "loss": 0.037302885204553604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03801, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.404407262802124, + "learning_rate": 9.999624706943868e-05, + "loss": 0.010177684016525745, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01023, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 1.1107022762298584, + "learning_rate": 9.999525021479544e-05, + "loss": 0.016156412661075592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01629, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 2.0084729194641113, + "learning_rate": 9.99941360918465e-05, + "loss": 0.022498758509755135, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02275, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.9237216711044312, + "learning_rate": 9.999290470349554e-05, + "loss": 0.009256366640329361, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0093, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 1.0012309551239014, + "learning_rate": 9.999155605295188e-05, + "loss": 0.01096038706600666, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01102, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 0.36062437295913696, + "learning_rate": 9.999009014373037e-05, + "loss": 0.0026956750079989433, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0027, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.09146630764007568, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0006923460750840604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00069, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 0.35573962330818176, + "learning_rate": 9.99868065648415e-05, + "loss": 0.0032197514083236456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00322, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.18824201822280884, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0015725638950243592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00157, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 2.2947869300842285, + "learning_rate": 9.998305400106009e-05, + "loss": 0.09606514871120453, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.10083, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.6851844787597656, + "learning_rate": 9.998100186186879e-05, + "loss": 0.006874916143715382, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0069, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.7458359003067017, + "learning_rate": 9.997883249150642e-05, + "loss": 0.020516997203230858, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02073, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 0.7431558966636658, + "learning_rate": 9.99765458956268e-05, + "loss": 0.014651929959654808, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01476, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.16357871890068054, + "learning_rate": 9.997414208018943e-05, + "loss": 0.003089388832449913, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00309, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.36700522899627686, + "learning_rate": 9.997162105145916e-05, + "loss": 0.01339892391115427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01349, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.22755062580108643, + "learning_rate": 9.996898281600642e-05, + "loss": 0.005444459617137909, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00546, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.339722216129303, + "learning_rate": 9.996622738070706e-05, + "loss": 0.004514522384852171, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00452, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 1.7278125286102295, + "learning_rate": 9.996335475274243e-05, + "loss": 0.024016786366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02431, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41380 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.326501019236718e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-128/training_args.bin b/aft/coin/checkpoint-128/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-128/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-16/README.md b/aft/coin/checkpoint-16/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-16/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-16/adapter_config.json b/aft/coin/checkpoint-16/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-16/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-16/adapter_model.safetensors b/aft/coin/checkpoint-16/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3eff7c0b7bcedfb36934f9c5110b593db63f254a --- /dev/null +++ b/aft/coin/checkpoint-16/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e0d19eaa90a1ffc79084d9f4eed449f753a0ce19e6c15db5970a2f0c64f835c +size 1047628488 diff --git a/aft/coin/checkpoint-16/chat_template.jinja b/aft/coin/checkpoint-16/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-16/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-16/tokenizer.json b/aft/coin/checkpoint-16/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-16/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-16/tokenizer_config.json b/aft/coin/checkpoint-16/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-16/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-16/tokens_state.json b/aft/coin/checkpoint-16/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b63988a0173f7326248f9087cfc197d335c083ba --- /dev/null +++ b/aft/coin/checkpoint-16/tokens_state.json @@ -0,0 +1 @@ +{"total": 241452, "trainable": 5168} \ No newline at end of file diff --git a/aft/coin/checkpoint-16/trainer_state.json b/aft/coin/checkpoint-16/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d9fbccc704966a0cbd63612182ac085352a27fe5 --- /dev/null +++ b/aft/coin/checkpoint-16/trainer_state.json @@ -0,0 +1,258 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.25, + "eval_steps": 500, + "global_step": 16, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6569802255655808e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-16/training_args.bin b/aft/coin/checkpoint-16/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-16/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-2048/README.md b/aft/coin/checkpoint-2048/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-2048/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-2048/adapter_config.json b/aft/coin/checkpoint-2048/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-2048/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-2048/adapter_model.safetensors b/aft/coin/checkpoint-2048/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c33cd819fda959d23b20557c5bfdc3a20c49c8b2 --- /dev/null +++ b/aft/coin/checkpoint-2048/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2291ff638a56ddfe4acea20f1ec0ea1b3d20f064c01dcaeaef8e3ccf5b3feaa0 +size 1047628488 diff --git a/aft/coin/checkpoint-2048/chat_template.jinja b/aft/coin/checkpoint-2048/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-2048/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-2048/tokenizer.json b/aft/coin/checkpoint-2048/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-2048/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-2048/tokenizer_config.json b/aft/coin/checkpoint-2048/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-2048/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-2048/tokens_state.json b/aft/coin/checkpoint-2048/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fbcb394b5828d5d9fe85c6ac8cb9d634da452676 --- /dev/null +++ b/aft/coin/checkpoint-2048/tokens_state.json @@ -0,0 +1 @@ +{"total": 30922344, "trainable": 662080} \ No newline at end of file diff --git a/aft/coin/checkpoint-2048/trainer_state.json b/aft/coin/checkpoint-2048/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f245870b3f82d992cc6bc3381deaba1b2b70c621 --- /dev/null +++ b/aft/coin/checkpoint-2048/trainer_state.json @@ -0,0 +1,28706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 32.0, + "eval_steps": 500, + "global_step": 2048, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0028458833694458, + "learning_rate": 6.274509803921569e-05, + "loss": 0.01862957514822483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0188, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.9001771807670593, + "learning_rate": 6.372549019607843e-05, + "loss": 0.021384725347161293, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02162, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.5960713028907776, + "learning_rate": 6.470588235294118e-05, + "loss": 0.010063373483717442, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01011, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 2.038512706756592, + "learning_rate": 6.568627450980392e-05, + "loss": 0.03831503912806511, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03906, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 2.5726208686828613, + "learning_rate": 6.666666666666667e-05, + "loss": 0.03134162724018097, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03184, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.793451189994812, + "learning_rate": 6.764705882352942e-05, + "loss": 0.028099995106458664, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0285, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 0.7262976169586182, + "learning_rate": 6.862745098039216e-05, + "loss": 0.02038084715604782, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02059, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 0.532096803188324, + "learning_rate": 6.96078431372549e-05, + "loss": 0.013697818852961063, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01379, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 0.7747638821601868, + "learning_rate": 7.058823529411765e-05, + "loss": 0.023866616189479828, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02415, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 0.49953746795654297, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014121961779892445, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01422, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.3340316116809845, + "learning_rate": 7.254901960784314e-05, + "loss": 0.012052969075739384, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01213, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.735864520072937, + "learning_rate": 7.352941176470589e-05, + "loss": 0.04014549404382706, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04096, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.4149185419082642, + "learning_rate": 7.450980392156864e-05, + "loss": 0.045495182275772095, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04655, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 0.31831473112106323, + "learning_rate": 7.549019607843137e-05, + "loss": 0.011568848043680191, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01164, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 0.5228955745697021, + "learning_rate": 7.647058823529411e-05, + "loss": 0.02045898698270321, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02067, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.5053377151489258, + "learning_rate": 7.745098039215687e-05, + "loss": 0.020432129502296448, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02064, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.35163524746894836, + "learning_rate": 7.843137254901961e-05, + "loss": 0.01403958909213543, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01414, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.6905986070632935, + "learning_rate": 7.941176470588235e-05, + "loss": 0.03329383581876755, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.03385, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 0.4459448456764221, + "learning_rate": 8.039215686274511e-05, + "loss": 0.006232030689716339, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00625, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.2638562321662903, + "learning_rate": 8.137254901960785e-05, + "loss": 0.006893642246723175, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00692, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 1.170509696006775, + "learning_rate": 8.23529411764706e-05, + "loss": 0.0398201048374176, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.04062, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.8102952837944031, + "learning_rate": 8.333333333333334e-05, + "loss": 0.018316198140382767, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01848, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.6812735795974731, + "learning_rate": 8.431372549019608e-05, + "loss": 0.02392837405204773, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02422, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.251705288887024, + "learning_rate": 8.529411764705883e-05, + "loss": 0.024196332320570946, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02449, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.5772960782051086, + "learning_rate": 8.627450980392158e-05, + "loss": 0.021560901775956154, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0218, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.6470917463302612, + "learning_rate": 8.725490196078432e-05, + "loss": 0.04491877555847168, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.04594, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.41440942883491516, + "learning_rate": 8.823529411764706e-05, + "loss": 0.0230612400919199, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02333, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 0.39720970392227173, + "learning_rate": 8.921568627450981e-05, + "loss": 0.013313441537320614, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0134, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.8898259401321411, + "learning_rate": 9.019607843137255e-05, + "loss": 0.015421045944094658, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01554, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 8.74992847442627, + "learning_rate": 9.11764705882353e-05, + "loss": 0.03271842002868652, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03326, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.18793171644210815, + "learning_rate": 9.215686274509804e-05, + "loss": 0.0015747153665870428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00158, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.9997888207435608, + "learning_rate": 9.313725490196079e-05, + "loss": 0.03334064409136772, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0339, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.6498851776123047, + "learning_rate": 9.411764705882353e-05, + "loss": 0.023218588903546333, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02349, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.5593264102935791, + "learning_rate": 9.509803921568627e-05, + "loss": 0.005530651658773422, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00555, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.8789476752281189, + "learning_rate": 9.607843137254903e-05, + "loss": 0.04554657265543938, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0466, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.5140370726585388, + "learning_rate": 9.705882352941177e-05, + "loss": 0.014101475477218628, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0142, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 0.5285096764564514, + "learning_rate": 9.80392156862745e-05, + "loss": 0.030611136928200722, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03108, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 0.6092981696128845, + "learning_rate": 9.901960784313727e-05, + "loss": 0.028998689725995064, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02942, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0235849618911743, + "learning_rate": 0.0001, + "loss": 0.025761116296052933, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0261, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.9383920431137085, + "learning_rate": 9.999994135965763e-05, + "loss": 0.02419213019311428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02449, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.8995917439460754, + "learning_rate": 9.999976543878332e-05, + "loss": 0.011181461624801159, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01124, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 3.9120519161224365, + "learning_rate": 9.999947223783559e-05, + "loss": 0.10826049745082855, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.11434, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 1.801079511642456, + "learning_rate": 9.999906175757857e-05, + "loss": 0.029263045638799667, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0297, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 3.040914535522461, + "learning_rate": 9.99985339990821e-05, + "loss": 0.04611361399292946, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04719, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.6034630537033081, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0116285290569067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0117, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.238577961921692, + "learning_rate": 9.999712665317819e-05, + "loss": 0.037302885204553604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03801, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.404407262802124, + "learning_rate": 9.999624706943868e-05, + "loss": 0.010177684016525745, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01023, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 1.1107022762298584, + "learning_rate": 9.999525021479544e-05, + "loss": 0.016156412661075592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01629, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 2.0084729194641113, + "learning_rate": 9.99941360918465e-05, + "loss": 0.022498758509755135, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02275, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.9237216711044312, + "learning_rate": 9.999290470349554e-05, + "loss": 0.009256366640329361, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0093, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 1.0012309551239014, + "learning_rate": 9.999155605295188e-05, + "loss": 0.01096038706600666, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01102, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 0.36062437295913696, + "learning_rate": 9.999009014373037e-05, + "loss": 0.0026956750079989433, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0027, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.09146630764007568, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0006923460750840604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00069, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 0.35573962330818176, + "learning_rate": 9.99868065648415e-05, + "loss": 0.0032197514083236456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00322, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.18824201822280884, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0015725638950243592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00157, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 2.2947869300842285, + "learning_rate": 9.998305400106009e-05, + "loss": 0.09606514871120453, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.10083, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.6851844787597656, + "learning_rate": 9.998100186186879e-05, + "loss": 0.006874916143715382, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0069, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.7458359003067017, + "learning_rate": 9.997883249150642e-05, + "loss": 0.020516997203230858, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02073, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 0.7431558966636658, + "learning_rate": 9.99765458956268e-05, + "loss": 0.014651929959654808, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01476, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.16357871890068054, + "learning_rate": 9.997414208018943e-05, + "loss": 0.003089388832449913, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00309, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.36700522899627686, + "learning_rate": 9.997162105145916e-05, + "loss": 0.01339892391115427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01349, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.22755062580108643, + "learning_rate": 9.996898281600642e-05, + "loss": 0.005444459617137909, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00546, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.339722216129303, + "learning_rate": 9.996622738070706e-05, + "loss": 0.004514522384852171, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00452, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 1.7278125286102295, + "learning_rate": 9.996335475274243e-05, + "loss": 0.024016786366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02431, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.47769469022750854, + "learning_rate": 9.996036493959926e-05, + "loss": 0.010395423509180546, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01045, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.4170791804790497, + "learning_rate": 9.99572579490697e-05, + "loss": 0.004119648598134518, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00413, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.37145885825157166, + "learning_rate": 9.995403378925132e-05, + "loss": 0.00920130219310522, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00924, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 0.06787586212158203, + "learning_rate": 9.995069246854705e-05, + "loss": 0.0012981402687728405, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.7351065874099731, + "learning_rate": 9.994723399566515e-05, + "loss": 0.021887682378292084, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02213, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.590771496295929, + "learning_rate": 9.994365837961926e-05, + "loss": 0.009705535136163235, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00975, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.3608993589878082, + "learning_rate": 9.993996562972823e-05, + "loss": 0.008060690015554428, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00809, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.48873215913772583, + "learning_rate": 9.993615575561626e-05, + "loss": 0.008283237926661968, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00832, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 0.7132001519203186, + "learning_rate": 9.99322287672128e-05, + "loss": 0.024729935452342033, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02504, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.09246033430099487, + "learning_rate": 9.992818467475254e-05, + "loss": 0.001992235891520977, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00199, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.7287611365318298, + "learning_rate": 9.992402348877528e-05, + "loss": 0.021320564672350883, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02155, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 0.2133086621761322, + "learning_rate": 9.991974522012612e-05, + "loss": 0.00337275885976851, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00338, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.37355223298072815, + "learning_rate": 9.99153498799552e-05, + "loss": 0.007823438383638859, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00785, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 0.06703885644674301, + "learning_rate": 9.991083747971783e-05, + "loss": 0.0005649570957757533, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00057, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.06440247595310211, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0006133198039606214, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00061, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 0.34707385301589966, + "learning_rate": 9.990146154639039e-05, + "loss": 0.0035533613990992308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00356, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 2.017024517059326, + "learning_rate": 9.98965980377362e-05, + "loss": 0.042595766484737396, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04352, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.785468339920044, + "learning_rate": 9.989161751788734e-05, + "loss": 0.002102769911289215, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0021, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 2.8202157020568848, + "learning_rate": 9.98865199998242e-05, + "loss": 0.01083361729979515, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01089, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 1.0493052005767822, + "learning_rate": 9.988130549683211e-05, + "loss": 0.014770902693271637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01488, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 2.157977819442749, + "learning_rate": 9.987597402250136e-05, + "loss": 0.009032780304551125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00907, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.029030824080109596, + "learning_rate": 9.987052559072696e-05, + "loss": 0.00013317995762918144, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 2.6754238605499268, + "learning_rate": 9.98649602157089e-05, + "loss": 0.033499784767627716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03407, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.9873471856117249, + "learning_rate": 9.985927791195179e-05, + "loss": 0.046224698424339294, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04731, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 1.0582185983657837, + "learning_rate": 9.985347869426513e-05, + "loss": 0.01951594091951847, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01971, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.6863466501235962, + "learning_rate": 9.984756257776302e-05, + "loss": 0.021039601415395737, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02126, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4057074785232544, + "learning_rate": 9.984152957786426e-05, + "loss": 0.008229494094848633, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00826, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.4019395411014557, + "learning_rate": 9.983537971029228e-05, + "loss": 0.010984211228787899, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01104, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.48016035556793213, + "learning_rate": 9.982911299107514e-05, + "loss": 0.012419218197464943, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.440166175365448, + "learning_rate": 9.982272943654535e-05, + "loss": 0.009114887565374374, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00916, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.20877902209758759, + "learning_rate": 9.981622906333999e-05, + "loss": 0.00270082987844944, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0027, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.6577605605125427, + "learning_rate": 9.980961188840055e-05, + "loss": 0.004442816600203514, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00445, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 0.13625678420066833, + "learning_rate": 9.9802877928973e-05, + "loss": 0.0019005294889211655, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0019, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.019028058275580406, + "learning_rate": 9.979602720260762e-05, + "loss": 0.0002803217794280499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00028, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.0662836953997612, + "learning_rate": 9.978905972715901e-05, + "loss": 0.0006815034430474043, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00068, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.9524247646331787, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00464291637763381, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00465, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 0.04438980668783188, + "learning_rate": 9.977477460195198e-05, + "loss": 0.00037421027082018554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 0.7781257629394531, + "learning_rate": 9.976745698942397e-05, + "loss": 0.0023766113445162773, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00238, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.4958593249320984, + "learning_rate": 9.976002270227351e-05, + "loss": 0.029030824080109596, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02946, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 2.34055495262146, + "learning_rate": 9.975247175987611e-05, + "loss": 0.04764559492468834, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0488, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.025128737092018127, + "learning_rate": 9.974480418191133e-05, + "loss": 0.0001649868063395843, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 2.489701986312866, + "learning_rate": 9.973701998836269e-05, + "loss": 0.17427006363868713, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.19038, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 1.9490054845809937, + "learning_rate": 9.972911919951763e-05, + "loss": 0.06718016415834427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06949, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 1.13898766040802, + "learning_rate": 9.972110183596752e-05, + "loss": 0.024798311293125153, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02511, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.4100293517112732, + "learning_rate": 9.971296791860748e-05, + "loss": 0.012426480650901794, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.5648359060287476, + "learning_rate": 9.970471746863644e-05, + "loss": 0.03443408012390137, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03503, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.3096520006656647, + "learning_rate": 9.969635050755704e-05, + "loss": 0.012037754990160465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.01211, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.42714670300483704, + "learning_rate": 9.968786705717555e-05, + "loss": 0.01219131425023079, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01227, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.5599139928817749, + "learning_rate": 9.967926713960188e-05, + "loss": 0.012130084447562695, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0122, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.6217107772827148, + "learning_rate": 9.967055077724944e-05, + "loss": 0.009348927065730095, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00939, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.0592292957007885, + "learning_rate": 9.966171799283513e-05, + "loss": 0.001284315949305892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00129, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 0.462398886680603, + "learning_rate": 9.965276880937933e-05, + "loss": 0.01244058832526207, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01252, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 8.241695404052734, + "learning_rate": 9.964370325020568e-05, + "loss": 0.05807769298553467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0598, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 1.0815411806106567, + "learning_rate": 9.96345213389412e-05, + "loss": 0.017220184206962585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01737, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 1.6311028003692627, + "learning_rate": 9.962522309951614e-05, + "loss": 0.005533762276172638, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00555, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 3.626565456390381, + "learning_rate": 9.961580855616392e-05, + "loss": 0.09052321314811707, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.09475, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 3.3724992275238037, + "learning_rate": 9.960627773342105e-05, + "loss": 0.00758603448048234, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00761, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 0.3157868981361389, + "learning_rate": 9.959663065612716e-05, + "loss": 0.003882938763126731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00389, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.03412550315260887, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0005760709755122662, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00058, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.52850741147995, + "learning_rate": 9.957698783875946e-05, + "loss": 0.010527489706873894, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01058, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.3949584364891052, + "learning_rate": 9.95669921498795e-05, + "loss": 0.007605592254549265, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00763, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 0.018618274480104446, + "learning_rate": 9.955688030883605e-05, + "loss": 0.0003067223879043013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00031, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.11938109248876572, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0008339773630723357, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00083, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 1.4441187381744385, + "learning_rate": 9.953630827597677e-05, + "loss": 0.015039840713143349, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01515, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.04352337867021561, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0008668563095852733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00087, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 10.194028854370117, + "learning_rate": 9.951527195464393e-05, + "loss": 0.01512847188860178, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01524, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.9922191500663757, + "learning_rate": 9.950457975414293e-05, + "loss": 0.012203281745314598, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01228, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 1.3960785865783691, + "learning_rate": 9.949377156413995e-05, + "loss": 0.010049878619611263, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0101, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.6242722272872925, + "learning_rate": 9.948284741280371e-05, + "loss": 0.020798590034246445, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02102, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.9250545501708984, + "learning_rate": 9.947180732860518e-05, + "loss": 0.02078751102089882, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02101, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.44395211338996887, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0027410993352532387, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00274, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.06538265198469162, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0013557756319642067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00136, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.8934104442596436, + "learning_rate": 9.943799176807672e-05, + "loss": 0.009296886622905731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00934, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.285216748714447, + "learning_rate": 9.942648824318006e-05, + "loss": 0.0030592025723308325, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00306, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.6151894330978394, + "learning_rate": 9.941486893230645e-05, + "loss": 0.00584901450201869, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00587, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.3476015329360962, + "learning_rate": 9.940313386573859e-05, + "loss": 0.005612781271338463, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00563, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.04463421553373337, + "learning_rate": 9.939128307406084e-05, + "loss": 0.0004844452196266502, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00048, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.14716975390911102, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0013532615266740322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00135, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.013975728303194046, + "learning_rate": 9.936723443922109e-05, + "loss": 0.00017458554066251963, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.23583655059337616, + "learning_rate": 9.935503665873555e-05, + "loss": 0.0013064221711829305, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00131, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 0.19946330785751343, + "learning_rate": 9.934272327849287e-05, + "loss": 0.0016659648390486836, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00167, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 2.5991828441619873, + "learning_rate": 9.933029433058463e-05, + "loss": 0.03332412242889404, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03389, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.022260449826717377, + "learning_rate": 9.931774984740362e-05, + "loss": 0.00019467048696242273, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00019, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 1.043959140777588, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0496981106698513, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05095, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.009332562796771526, + "learning_rate": 9.929231440629993e-05, + "loss": 0.0001336124842055142, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00013, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.028200708329677582, + "learning_rate": 9.927942351466806e-05, + "loss": 0.00035777001176029444, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00036, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.06076425686478615, + "learning_rate": 9.926641722034484e-05, + "loss": 0.001383051392622292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00138, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.12561249732971191, + "learning_rate": 9.925329555722777e-05, + "loss": 0.0026724478229880333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00268, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 0.5528528094291687, + "learning_rate": 9.924005855951503e-05, + "loss": 0.01256745308637619, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01265, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 2.376209259033203, + "learning_rate": 9.922670626170537e-05, + "loss": 0.017799941822886467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01796, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.2762843370437622, + "learning_rate": 9.921323869859805e-05, + "loss": 0.007115472108125687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00714, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.24070966243743896, + "learning_rate": 9.919965590529272e-05, + "loss": 0.0015583797357976437, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00156, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.10395435988903046, + "learning_rate": 9.918595791718938e-05, + "loss": 0.0007914248271845281, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00079, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.5730319023132324, + "learning_rate": 9.917214476998825e-05, + "loss": 0.005572602618485689, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00559, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.1355881541967392, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0004524950636550784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00045, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.07415846735239029, + "learning_rate": 9.914417314259398e-05, + "loss": 0.0004993464681319892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0005, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 0.22872294485569, + "learning_rate": 9.913001473530153e-05, + "loss": 0.0009260782971978188, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00093, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.006719402503222227, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001365313510177657, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.020894810557365417, + "learning_rate": 9.910135291802682e-05, + "loss": 0.00035784178180620074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00036, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.06920765340328217, + "learning_rate": 9.908684958274405e-05, + "loss": 0.000399862794438377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0004, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.008416767232120037, + "learning_rate": 9.907223134666332e-05, + "loss": 4.7169265599222854e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00005, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 0.013850455172359943, + "learning_rate": 9.905749824788326e-05, + "loss": 0.00019475985027384013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00019, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.0023701407480984926, + "learning_rate": 9.904265032480177e-05, + "loss": 3.626237958087586e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.0015644921222701669, + "learning_rate": 9.902768761611609e-05, + "loss": 3.6560850276146084e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 0.54908686876297, + "learning_rate": 9.901261016082262e-05, + "loss": 0.02532411366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02565, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.00214971462264657, + "learning_rate": 9.899741799821673e-05, + "loss": 3.0114068067632616e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0071777780540287495, + "learning_rate": 9.898211116789284e-05, + "loss": 5.1465347496559843e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 1.685914158821106, + "learning_rate": 9.896668970974419e-05, + "loss": 0.014849710278213024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01496, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 1.67842698097229, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0018469173228368163, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.005261159967631102, + "learning_rate": 9.893550307103913e-05, + "loss": 8.17667314549908e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.023177370429039, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00030886579770594835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00031, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.3134678304195404, + "learning_rate": 9.890385840722048e-05, + "loss": 0.0008985826279968023, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0009, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.7450703382492065, + "learning_rate": 9.888786441879897e-05, + "loss": 0.017146039754152298, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01729, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 0.008559435606002808, + "learning_rate": 9.887175604818206e-05, + "loss": 0.0001242949510924518, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.00012, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 1.7312885522842407, + "learning_rate": 9.885553333735204e-05, + "loss": 0.008018076419830322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00805, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 1.6031489372253418, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0027930131182074547, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0028, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.7863869071006775, + "learning_rate": 9.882274506447146e-05, + "loss": 0.02333652786910534, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02361, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.6194532513618469, + "learning_rate": 9.88061795878749e-05, + "loss": 0.003525766544044018, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00353, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 1.360031008720398, + "learning_rate": 9.878949994197307e-05, + "loss": 0.017261315137147903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01741, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03367060795426369, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0006716687930747867, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00067, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.1465068757534027, + "learning_rate": 9.875579831643538e-05, + "loss": 0.0024646068923175335, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00247, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.6451879739761353, + "learning_rate": 9.873877642463397e-05, + "loss": 0.016013173386454582, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01614, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.5427649617195129, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010463962331414223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01052, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.14428693056106567, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0027021225541830063, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00271, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.41795286536216736, + "learning_rate": 9.868702696634756e-05, + "loss": 0.014849536120891571, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01496, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.33399561047554016, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0074952589347958565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00752, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.2544633150100708, + "learning_rate": 9.865195795873484e-05, + "loss": 0.005140242632478476, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00515, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.6971493363380432, + "learning_rate": 9.863425278095409e-05, + "loss": 0.02200455032289028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02225, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.15962868928909302, + "learning_rate": 9.861643388195015e-05, + "loss": 0.002751844236627221, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00276, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.3089817464351654, + "learning_rate": 9.859850130816326e-05, + "loss": 0.0019627867732197046, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00196, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.03729058429598808, + "learning_rate": 9.858045510632996e-05, + "loss": 0.0004204344586469233, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00042, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.2555536925792694, + "learning_rate": 9.856229532348296e-05, + "loss": 0.002585819223895669, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00259, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.017389044165611267, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00010104697139468044, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 0.002870214870199561, + "learning_rate": 9.852563520435858e-05, + "loss": 5.9336227423045784e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.2427501380443573, + "learning_rate": 9.850713496362614e-05, + "loss": 0.004451706539839506, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00446, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.005748335737735033, + "learning_rate": 9.848852133296972e-05, + "loss": 8.568367775296792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0012240263167768717, + "learning_rate": 9.846979436090085e-05, + "loss": 4.23779092670884e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.01661101169884205, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0001144747220678255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.020272312685847282, + "learning_rate": 9.843200058804868e-05, + "loss": 0.00017771853890735656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00018, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.033239174634218216, + "learning_rate": 9.841293388576495e-05, + "loss": 0.00044481243821792305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00044, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5434594750404358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.009829508140683174, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00988, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.3308855891227722, + "learning_rate": 9.837446109794382e-05, + "loss": 0.004846261814236641, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00486, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.003991277422755957, + "learning_rate": 9.835505511267567e-05, + "loss": 8.371799776796252e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.10116580128669739, + "learning_rate": 9.83355361338397e-05, + "loss": 0.0005483412533067167, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00055, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.003369405400007963, + "learning_rate": 9.831590421230704e-05, + "loss": 7.847918459447101e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00008, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.011247020214796066, + "learning_rate": 9.829615939924308e-05, + "loss": 0.000140060976264067, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.009074223227798939, + "learning_rate": 9.827630174610754e-05, + "loss": 9.554257121635601e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0001, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.000541043933480978, + "learning_rate": 9.825633130465418e-05, + "loss": 1.697065818007104e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.014442048035562038, + "learning_rate": 9.823624812693069e-05, + "loss": 9.099359886022285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00009, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.013665883801877499, + "learning_rate": 9.821605226527862e-05, + "loss": 0.00011380278010619804, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00011, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.0006537620793096721, + "learning_rate": 9.819574377233317e-05, + "loss": 1.6407084331149235e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.01291027944535017, + "learning_rate": 9.817532270102309e-05, + "loss": 6.133162969490513e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00006, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.5513811111450195, + "learning_rate": 9.815478910457054e-05, + "loss": 0.007841670885682106, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00787, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.0005016659270040691, + "learning_rate": 9.813414303649097e-05, + "loss": 1.6098865671665408e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 0.1864936798810959, + "learning_rate": 9.811338455059291e-05, + "loss": 0.0015166733646765351, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00152, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.0026293513365089893, + "learning_rate": 9.809251370097791e-05, + "loss": 5.2753948693862185e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00005, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.41635486483573914, + "learning_rate": 9.807153054204036e-05, + "loss": 0.02190607599914074, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02215, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 1.0917211771011353, + "learning_rate": 9.805043512846735e-05, + "loss": 0.019963297992944717, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02016, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.06857825815677643, + "learning_rate": 9.802922751523855e-05, + "loss": 0.00030228254036046565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.031100548803806305, + "learning_rate": 9.800790775762605e-05, + "loss": 0.0004046057292725891, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0004, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.47315219044685364, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0014745643129572272, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00148, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.0773354172706604, + "learning_rate": 9.796493203179937e-05, + "loss": 0.0008912066696211696, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00089, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.1289902925491333, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0018994644051417708, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0019, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.054183945059776306, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0005117067485116422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00051, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.12445089966058731, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0022513207513839006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00225, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 9.98, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.20701012015342712, + "learning_rate": 9.787763731193733e-05, + "loss": 0.003331368323415518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00334, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.6194630265235901, + "learning_rate": 9.785553411578963e-05, + "loss": 0.03162220120429993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03213, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.2134065181016922, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0037704987917095423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00378, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.040715303272008896, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0002603174070827663, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00026, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 0.7002236247062683, + "learning_rate": 9.778855460904167e-05, + "loss": 0.006636213511228561, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00666, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.007663452997803688, + "learning_rate": 9.77660049946513e-05, + "loss": 0.00024576610303483903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00025, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.023272940889000893, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0005693582352250814, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00057, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.026790525764226913, + "learning_rate": 9.77205714498407e-05, + "loss": 0.0007213435601443052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.004755170550197363, + "learning_rate": 9.769768763783107e-05, + "loss": 0.00017550830671098083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00018, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 0.016630757600069046, + "learning_rate": 9.767469254550936e-05, + "loss": 0.0002396691415924579, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00024, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 0.09570515155792236, + "learning_rate": 9.765158623280625e-05, + "loss": 0.0009529241942800581, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00095, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.5903910994529724, + "learning_rate": 9.762836875994225e-05, + "loss": 0.005048507358878851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00506, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.008239318616688251, + "learning_rate": 9.760504018742764e-05, + "loss": 0.00023036239144857973, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00023, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.060113124549388885, + "learning_rate": 9.75816005760622e-05, + "loss": 0.0003171982243657112, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00032, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.008316988125443459, + "learning_rate": 9.755804998693513e-05, + "loss": 0.00014665098569821566, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00015, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.21268479526042938, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0025118833873420954, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00252, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.005383700132369995, + "learning_rate": 9.751061612119887e-05, + "loss": 0.00013681002019438893, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00014, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.4226222336292267, + "learning_rate": 9.748673296821358e-05, + "loss": 0.00028868031222373247, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00029, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.007631849963217974, + "learning_rate": 9.746273908471418e-05, + "loss": 0.000154903216753155, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.09700846672058105, + "learning_rate": 9.743863453323441e-05, + "loss": 0.0013645484577864408, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.018550626933574677, + "learning_rate": 9.741441937659645e-05, + "loss": 0.0002988259366247803, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0003, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.005299921613186598, + "learning_rate": 9.73900936779108e-05, + "loss": 8.563048322685063e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.015369605273008347, + "learning_rate": 9.736565750057593e-05, + "loss": 0.00014232251851353794, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.006885336712002754, + "learning_rate": 9.734111090827836e-05, + "loss": 0.00014437930076383054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00014, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.3322601020336151, + "learning_rate": 9.731645396499232e-05, + "loss": 0.0058242324739694595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00584, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.008417945355176926, + "learning_rate": 9.729168673497969e-05, + "loss": 9.954217239283025e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0001, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.3274463713169098, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0031569483689963818, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00316, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.011693589389324188, + "learning_rate": 9.724182167325899e-05, + "loss": 7.778844883432612e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00008, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 1.3683384656906128, + "learning_rate": 9.721672397151109e-05, + "loss": 0.010511158034205437, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01057, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.005915132351219654, + "learning_rate": 9.719151624295665e-05, + "loss": 6.681385275442153e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00007, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.00782559160143137, + "learning_rate": 9.716619855329296e-05, + "loss": 8.550586790079251e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00009, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.0015690920408815145, + "learning_rate": 9.714077096850394e-05, + "loss": 2.1982301404932514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.0027190272230654955, + "learning_rate": 9.711523355485992e-05, + "loss": 2.6376750611234456e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 0.21495184302330017, + "learning_rate": 9.708958637891746e-05, + "loss": 0.0006077390280552208, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 0.0004555387713480741, + "learning_rate": 9.706382950751918e-05, + "loss": 1.7097307136282325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 0.02135647088289261, + "learning_rate": 9.703796300779361e-05, + "loss": 0.00014291358820628375, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 0.011736172251403332, + "learning_rate": 9.7011986947155e-05, + "loss": 3.78161967091728e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00004, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.874617338180542, + "learning_rate": 9.698590139330309e-05, + "loss": 0.006245121825486422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00626, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 0.027552802115678787, + "learning_rate": 9.695970641422309e-05, + "loss": 0.00019723488367162645, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 0.00035697652492672205, + "learning_rate": 9.693340207818528e-05, + "loss": 1.3179367670090869e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 0.08428651839494705, + "learning_rate": 9.690698845374505e-05, + "loss": 0.0008772820001468062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00088, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 0.00604958226904273, + "learning_rate": 9.688046560974256e-05, + "loss": 8.847641583997756e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 0.00039003140409477055, + "learning_rate": 9.685383361530262e-05, + "loss": 1.5304913176805712e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 0.00023623408924322575, + "learning_rate": 9.682709253983455e-05, + "loss": 8.844612239045091e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.00048069472541101277, + "learning_rate": 9.680024245303193e-05, + "loss": 1.667063588683959e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.16455329954624176, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0005412459140643477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00054, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.012845818884670734, + "learning_rate": 9.674621552561782e-05, + "loss": 0.00013661464618053287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.7110337615013123, + "learning_rate": 9.671903882581331e-05, + "loss": 0.005711458623409271, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00573, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.007893380708992481, + "learning_rate": 9.669175339628791e-05, + "loss": 9.75146540440619e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.10780492424964905, + "learning_rate": 9.66643593081539e-05, + "loss": 0.0005473099881783128, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00055, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.0010501379147171974, + "learning_rate": 9.663685663280679e-05, + "loss": 2.7556767236092128e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.288404256105423, + "learning_rate": 9.66092454419251e-05, + "loss": 0.0018175432924181223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00182, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 1.9734091758728027, + "learning_rate": 9.65815258074701e-05, + "loss": 0.03248640149831772, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03302, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 0.2764054536819458, + "learning_rate": 9.655369780168579e-05, + "loss": 0.0009657267364673316, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00097, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.0005600836011581123, + "learning_rate": 9.652576149709852e-05, + "loss": 1.6435071302112192e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 0.0008129188208840787, + "learning_rate": 9.649771696651697e-05, + "loss": 2.3179851268650964e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.0014548521721735597, + "learning_rate": 9.646956428303183e-05, + "loss": 1.6358961147489026e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.5601457953453064, + "learning_rate": 9.644130352001567e-05, + "loss": 0.003099913941696286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0031, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.00297771324403584, + "learning_rate": 9.641293475112275e-05, + "loss": 4.133052425459027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.0016736306715756655, + "learning_rate": 9.638445805028884e-05, + "loss": 4.893036748399027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.001026933896355331, + "learning_rate": 9.635587349173095e-05, + "loss": 2.9338965759961866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.008905365131795406, + "learning_rate": 9.632718114994726e-05, + "loss": 9.410006168764085e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.44935929775238037, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0031317835673689842, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00314, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.004555090330541134, + "learning_rate": 9.626947341609934e-05, + "loss": 0.00011855485354317352, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00012, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.001394708058796823, + "learning_rate": 9.624045817443519e-05, + "loss": 4.678318146034144e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.0037189542781561613, + "learning_rate": 9.621133545034491e-05, + "loss": 0.00011213941616006196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.1473202258348465, + "learning_rate": 9.61821053197293e-05, + "loss": 0.0005101395072415471, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00051, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.012220866046845913, + "learning_rate": 9.615276785876898e-05, + "loss": 0.0002749463019426912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00027, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.007556390482932329, + "learning_rate": 9.612332314392435e-05, + "loss": 0.00013154096086509526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.002842485439032316, + "learning_rate": 9.609377125193535e-05, + "loss": 7.768635987304151e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00008, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.04467956721782684, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00022737993276678026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 1.0502334833145142, + "learning_rate": 9.603434624488023e-05, + "loss": 0.015947995707392693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01608, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.014036186039447784, + "learning_rate": 9.600447328468982e-05, + "loss": 0.00013877129822503775, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.6235236525535583, + "learning_rate": 9.597449345710597e-05, + "loss": 0.015155073255300522, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01527, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.0999632477760315, + "learning_rate": 9.594440684026324e-05, + "loss": 0.0012880185386165977, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00129, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.02786194533109665, + "learning_rate": 9.591421351257449e-05, + "loss": 0.0003391870704945177, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00034, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.5290789008140564, + "learning_rate": 9.588391355273066e-05, + "loss": 0.007406376302242279, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00743, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.01936807669699192, + "learning_rate": 9.585350703970069e-05, + "loss": 0.00025529557024128735, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00026, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 0.005768527276813984, + "learning_rate": 9.582299405273118e-05, + "loss": 8.689028618391603e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.9624814391136169, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00811285525560379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00815, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.018384141847491264, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0003791853378061205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00038, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.6165330410003662, + "learning_rate": 9.573081704481236e-05, + "loss": 0.013609700836241245, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0137, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.0840073898434639, + "learning_rate": 9.569987896009734e-05, + "loss": 0.00045918801333755255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00046, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.008723951876163483, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00011756666208384559, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00012, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.012961777858436108, + "learning_rate": 9.563768465093079e-05, + "loss": 0.00030332955066114664, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0003, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.02426735870540142, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00048618094297125936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00049, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.1119721531867981, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0008597768610343337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00086, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.03705863282084465, + "learning_rate": 9.554359905560886e-05, + "loss": 0.0003950979153160006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0004, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.6851837038993835, + "learning_rate": 9.551202574875235e-05, + "loss": 0.011227660812437534, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01129, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.06478452682495117, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0012987275840714574, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.886103630065918, + "learning_rate": 9.544856246572735e-05, + "loss": 0.019783541560173035, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01998, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.03494488075375557, + "learning_rate": 9.541667265495922e-05, + "loss": 0.0006417686236090958, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.1211373507976532, + "learning_rate": 9.538467750874569e-05, + "loss": 0.0016129965661093593, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00161, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.9197566509246826, + "learning_rate": 9.535257711047364e-05, + "loss": 0.01892932318150997, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01911, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.3712734878063202, + "learning_rate": 9.532037154380439e-05, + "loss": 0.004730743821710348, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00474, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.13118499517440796, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0026065458077937365, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00261, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.41263192892074585, + "learning_rate": 9.525564524128947e-05, + "loss": 0.02503504976630211, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02535, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.05777617543935776, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0012999713653698564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0013, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.16045935451984406, + "learning_rate": 9.519049927596896e-05, + "loss": 0.006284795701503754, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0063, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.10557752847671509, + "learning_rate": 9.515776913181807e-05, + "loss": 0.0018496124539524317, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00185, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.1812046468257904, + "learning_rate": 9.512493432698582e-05, + "loss": 0.003738866187632084, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00375, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.11412262171506882, + "learning_rate": 9.509199494704748e-05, + "loss": 0.00221292139030993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00222, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.05267500877380371, + "learning_rate": 9.505895107785091e-05, + "loss": 0.0010225172154605389, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00102, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.23904849588871002, + "learning_rate": 9.502580280551627e-05, + "loss": 0.005414942279458046, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00543, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.013456333428621292, + "learning_rate": 9.499255021643582e-05, + "loss": 0.00013312777446117252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00013, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.0178237147629261, + "learning_rate": 9.495919339727372e-05, + "loss": 0.0003349714388605207, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00034, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.049912769347429276, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0004279202257748693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.014288226142525673, + "learning_rate": 9.489216741671916e-05, + "loss": 0.00036869742325507104, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00037, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 0.00872453860938549, + "learning_rate": 9.485849843001232e-05, + "loss": 6.138686876511201e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.0013658624375239015, + "learning_rate": 9.48247255625946e-05, + "loss": 3.532581831677817e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.0017636138945817947, + "learning_rate": 9.479084890248611e-05, + "loss": 2.53205780609278e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.000683065562043339, + "learning_rate": 9.475686853797749e-05, + "loss": 1.986018833122216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.016209598630666733, + "learning_rate": 9.472278455762962e-05, + "loss": 0.00014525593724101782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00015, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.020629821345210075, + "learning_rate": 9.468859705027345e-05, + "loss": 8.353606972377747e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.0009277141070924699, + "learning_rate": 9.465430610500977e-05, + "loss": 1.4405310139409266e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.0002464439603500068, + "learning_rate": 9.461991181120889e-05, + "loss": 9.015106115839444e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.000667985004838556, + "learning_rate": 9.458541425851051e-05, + "loss": 2.003746703849174e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.002300550462678075, + "learning_rate": 9.455081353682347e-05, + "loss": 4.4835043809143826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00004, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.0017511912155896425, + "learning_rate": 9.451610973632543e-05, + "loss": 6.716565621900372e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.0028233909979462624, + "learning_rate": 9.448130294746276e-05, + "loss": 2.4473858502460644e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.00019360017904546112, + "learning_rate": 9.444639326095022e-05, + "loss": 5.723140020563733e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 0.00023336702724918723, + "learning_rate": 9.441138076777071e-05, + "loss": 4.722754511021776e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.0003432357043493539, + "learning_rate": 9.437626555917512e-05, + "loss": 9.122491974267177e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.23666708171367645, + "learning_rate": 9.434104772668203e-05, + "loss": 0.0011873405892401934, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00119, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.0001242521102540195, + "learning_rate": 9.430572736207746e-05, + "loss": 2.8345539249130525e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.00043305609142407775, + "learning_rate": 9.427030455741466e-05, + "loss": 1.0893121725530364e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005291244946420193, + "learning_rate": 9.423477940501391e-05, + "loss": 3.527114313328639e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00004, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0002391930902376771, + "learning_rate": 9.419915199746214e-05, + "loss": 4.351905317889759e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.0001795155694708228, + "learning_rate": 9.416342242761287e-05, + "loss": 4.172787157585844e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.00027239706832915545, + "learning_rate": 9.412759078858583e-05, + "loss": 6.559062967426144e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.004924969747662544, + "learning_rate": 9.409165717376678e-05, + "loss": 1.9311535652377643e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00002, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00028949262923561037, + "learning_rate": 9.405562167680723e-05, + "loss": 5.919015166000463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.0019513516454026103, + "learning_rate": 9.401948439162429e-05, + "loss": 6.4454043240402825e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.00024267581466119736, + "learning_rate": 9.398324541240028e-05, + "loss": 2.6868240183830494e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.0005200989544391632, + "learning_rate": 9.394690483358256e-05, + "loss": 5.118726221553516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.00012520125892478973, + "learning_rate": 9.391046274988333e-05, + "loss": 2.580744194347062e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.00037211537710390985, + "learning_rate": 9.387391925627933e-05, + "loss": 6.75070896249963e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.17, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.006821396294981241, + "learning_rate": 9.383727444801156e-05, + "loss": 1.0072410987049807e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.005923285149037838, + "learning_rate": 9.380052842058508e-05, + "loss": 1.7001904780045152e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0007364701014012098, + "learning_rate": 9.376368126976879e-05, + "loss": 6.04315209784545e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.0021281663794070482, + "learning_rate": 9.37267330915951e-05, + "loss": 4.2355783079983667e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.00013200279499869794, + "learning_rate": 9.368968398235973e-05, + "loss": 2.4993578335852362e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.62, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 9.607620449969545e-05, + "learning_rate": 9.365253403862147e-05, + "loss": 2.0314462290116353e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.00013611785834655166, + "learning_rate": 9.361528335720188e-05, + "loss": 2.2868246105645085e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0671246275305748, + "learning_rate": 9.35779320351851e-05, + "loss": 5.1696148148039356e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.00018044775060843676, + "learning_rate": 9.354048016991751e-05, + "loss": 3.048619419132592e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.00013236024824436754, + "learning_rate": 9.35029278590076e-05, + "loss": 3.4778872759488877e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.0, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.004715391434729099, + "learning_rate": 9.346527520032561e-05, + "loss": 2.927492005255772e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.00028085883241146803, + "learning_rate": 9.342752229200325e-05, + "loss": 4.623073891707463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 8.980643178801984e-05, + "learning_rate": 9.338966923243363e-05, + "loss": 1.8482477344150539e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.003043609904125333, + "learning_rate": 9.335171612027078e-05, + "loss": 1.538229662401136e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 8.966683526523411e-05, + "learning_rate": 9.331366305442954e-05, + "loss": 2.0870693333563395e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.0009604596416465938, + "learning_rate": 9.32755101340852e-05, + "loss": 3.408649718039669e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.00011633319081738591, + "learning_rate": 9.323725745867336e-05, + "loss": 3.0257524485932663e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.0010066182585433125, + "learning_rate": 9.319890512788957e-05, + "loss": 8.530637387593742e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.00016301196592394263, + "learning_rate": 9.316045324168912e-05, + "loss": 2.349399892409565e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.003284434787929058, + "learning_rate": 9.312190190028672e-05, + "loss": 2.500721711840015e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.00014885724522173405, + "learning_rate": 9.308325120415636e-05, + "loss": 3.3532028282934334e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 9.280376980314031e-05, + "learning_rate": 9.30445012540309e-05, + "loss": 1.334464627689158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 8.563206938561052e-05, + "learning_rate": 9.300565215090192e-05, + "loss": 1.549241460452322e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.00010646737791830674, + "learning_rate": 9.296670399601942e-05, + "loss": 1.6117182894959114e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 9.888842760119587e-05, + "learning_rate": 9.292765689089152e-05, + "loss": 1.9165411231369944e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 7.287397602340207e-05, + "learning_rate": 9.288851093728427e-05, + "loss": 2.075230213449686e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.00041501791565679014, + "learning_rate": 9.284926623722131e-05, + "loss": 3.292028623036458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 5.636964851873927e-05, + "learning_rate": 9.280992289298363e-05, + "loss": 1.1177535270689987e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.0004036540340166539, + "learning_rate": 9.277048100710934e-05, + "loss": 4.003003596153576e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 5.150034121470526e-05, + "learning_rate": 9.273094068239336e-05, + "loss": 1.1050038892790326e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.0008515650406479836, + "learning_rate": 9.269130202188718e-05, + "loss": 9.22057733987458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.818722128868103, + "learning_rate": 9.265156512889854e-05, + "loss": 0.0029519246891140938, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00296, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 7.819741585990414e-05, + "learning_rate": 9.261173010699122e-05, + "loss": 2.333828206246835e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.0005888771847821772, + "learning_rate": 9.257179705998475e-05, + "loss": 1.9795390926447e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 4.292320954846218e-05, + "learning_rate": 9.253176609195413e-05, + "loss": 1.0174778708460508e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 4.8053876525955275e-05, + "learning_rate": 9.249163730722958e-05, + "loss": 1.1561764949874487e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.00020360939379315823, + "learning_rate": 9.245141081039623e-05, + "loss": 2.147836312360596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 5.6159336963901296e-05, + "learning_rate": 9.241108670629387e-05, + "loss": 1.3456724445859436e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.4064067304134369, + "learning_rate": 9.23706651000167e-05, + "loss": 0.0016359895234927535, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00164, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.00012425552995409817, + "learning_rate": 9.233014609691304e-05, + "loss": 1.811459014788852e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.001742662163451314, + "learning_rate": 9.2289529802585e-05, + "loss": 2.9671325592062203e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.0006759302341379225, + "learning_rate": 9.224881632288832e-05, + "loss": 5.512976258614799e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.00019010427058674395, + "learning_rate": 9.2208005763932e-05, + "loss": 1.384292886541516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.002026048256084323, + "learning_rate": 9.2167098232078e-05, + "loss": 1.1068831554439384e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 4.712811278295703e-05, + "learning_rate": 9.21260938339411e-05, + "loss": 8.564664426558011e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.000333633302943781, + "learning_rate": 9.208499267638847e-05, + "loss": 4.601421096595004e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 4.1058650822378695e-05, + "learning_rate": 9.204379486653952e-05, + "loss": 7.030776600913669e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 4.685380918090232e-05, + "learning_rate": 9.20025005117655e-05, + "loss": 8.994193194666877e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 4.656754754250869e-05, + "learning_rate": 9.19611097196893e-05, + "loss": 1.0813282642629929e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 1.0787988901138306, + "learning_rate": 9.191962259818518e-05, + "loss": 0.024499906226992607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0248, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.00020449870498850942, + "learning_rate": 9.18780392553784e-05, + "loss": 2.7060591492045205e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.0011033477494493127, + "learning_rate": 9.183635979964503e-05, + "loss": 9.57420161284972e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.002774628810584545, + "learning_rate": 9.179458433961166e-05, + "loss": 3.007879786309786e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.026837196201086044, + "learning_rate": 9.1752712984155e-05, + "loss": 0.0002948661567643285, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00029, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.01849069632589817, + "learning_rate": 9.17107458424018e-05, + "loss": 0.00011084417928941548, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.009761124849319458, + "learning_rate": 9.166868302372835e-05, + "loss": 3.652252780739218e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.08665445446968079, + "learning_rate": 9.162652463776038e-05, + "loss": 0.0008891914039850235, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.03040151111781597, + "learning_rate": 9.158427079437264e-05, + "loss": 0.00030500703724101186, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.05672994256019592, + "learning_rate": 9.154192160368866e-05, + "loss": 0.0007530992152169347, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.003426315262913704, + "learning_rate": 9.149947717608052e-05, + "loss": 1.5134808563743718e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.004125513136386871, + "learning_rate": 9.145693762216845e-05, + "loss": 6.913553079357371e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00007, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.04262302443385124, + "learning_rate": 9.141430305282065e-05, + "loss": 0.000580799242015928, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00058, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 0.013481490314006805, + "learning_rate": 9.137157357915291e-05, + "loss": 8.695045835338533e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00009, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 4.995663166046143, + "learning_rate": 9.132874931252841e-05, + "loss": 0.02319028601050377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02346, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.0015562125481665134, + "learning_rate": 9.128583036455732e-05, + "loss": 2.1520958398468792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.0061469124630093575, + "learning_rate": 9.124281684709663e-05, + "loss": 8.727848035050556e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00009, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.003409374738112092, + "learning_rate": 9.119970887224978e-05, + "loss": 5.775516910944134e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00006, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.015050097368657589, + "learning_rate": 9.11565065523664e-05, + "loss": 0.0001365312928101048, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.01874382048845291, + "learning_rate": 9.111321000004195e-05, + "loss": 0.00020053039770573378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.002161014126613736, + "learning_rate": 9.106981932811758e-05, + "loss": 3.2826297683641315e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.0013815263519063592, + "learning_rate": 9.102633464967965e-05, + "loss": 3.158616891596466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.13355329632759094, + "learning_rate": 9.098275607805957e-05, + "loss": 0.00038878218038007617, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00039, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.04, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 0.0038680124562233686, + "learning_rate": 9.093908372683343e-05, + "loss": 7.378710870398208e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.3629514276981354, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0006530214450322092, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0069854846224188805, + "learning_rate": 9.085145814108921e-05, + "loss": 3.534342613420449e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0013717467663809657, + "learning_rate": 9.08075051349442e-05, + "loss": 2.084069637930952e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.010476332157850266, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0001456337922718376, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00015, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.002277064835652709, + "learning_rate": 9.071931926886792e-05, + "loss": 2.8683749405900016e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00003, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.4141025245189667, + "learning_rate": 9.067508663876997e-05, + "loss": 0.0020133755169808865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00202, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.001090523204766214, + "learning_rate": 9.063076103092557e-05, + "loss": 2.1892830773140304e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 0.0011893577175214887, + "learning_rate": 9.05863425608578e-05, + "loss": 1.7626425687922165e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.3477942645549774, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002018592320382595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00202, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 165520 + }, + { + "epoch": 8.015625, + "grad_norm": 0.0017783264629542828, + "learning_rate": 9.049722749735415e-05, + "loss": 1.2956855243828613e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 513, + "tokens/total": 7745176, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 165842 + }, + { + "epoch": 8.03125, + "grad_norm": 0.0009507329086773098, + "learning_rate": 9.04525311361733e-05, + "loss": 1.1546111636562273e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 514, + "tokens/total": 7760316, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 166164 + }, + { + "epoch": 8.046875, + "grad_norm": 0.013517077080905437, + "learning_rate": 9.04077423772785e-05, + "loss": 6.299532833509147e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00006, + "step": 515, + "tokens/total": 7775516, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 166490 + }, + { + "epoch": 8.0625, + "grad_norm": 0.0008650442468933761, + "learning_rate": 9.036286133739987e-05, + "loss": 2.090480484184809e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 516, + "tokens/total": 7790668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 166814 + }, + { + "epoch": 8.078125, + "grad_norm": 0.03777744248509407, + "learning_rate": 9.031788813350807e-05, + "loss": 0.00017636040865909308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00018, + "step": 517, + "tokens/total": 7805712, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 167137 + }, + { + "epoch": 8.09375, + "grad_norm": 0.06616051495075226, + "learning_rate": 9.027282288281396e-05, + "loss": 0.00020225218031555414, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 518, + "tokens/total": 7820848, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 167461 + }, + { + "epoch": 8.109375, + "grad_norm": 0.0019223307026550174, + "learning_rate": 9.022766570276829e-05, + "loss": 1.723412970022764e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 519, + "tokens/total": 7835976, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 167784 + }, + { + "epoch": 8.125, + "grad_norm": 0.0008735855808481574, + "learning_rate": 9.018241671106134e-05, + "loss": 1.880908530438319e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 520, + "tokens/total": 7851168, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 168106 + }, + { + "epoch": 8.140625, + "grad_norm": 0.37315821647644043, + "learning_rate": 9.013707602562279e-05, + "loss": 0.0009505169582553208, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00095, + "step": 521, + "tokens/total": 7866244, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 168434 + }, + { + "epoch": 8.15625, + "grad_norm": 0.00021756299247499555, + "learning_rate": 9.009164376462119e-05, + "loss": 7.845479558454826e-06, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 522, + "tokens/total": 7881132, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 168757 + }, + { + "epoch": 8.171875, + "grad_norm": 0.002056313678622246, + "learning_rate": 9.004612004646383e-05, + "loss": 2.414161281194538e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 523, + "tokens/total": 7896264, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 169077 + }, + { + "epoch": 8.1875, + "grad_norm": 0.06747874617576599, + "learning_rate": 9.00005049897963e-05, + "loss": 0.00028232645126990974, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00028, + "step": 524, + "tokens/total": 7911500, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 169403 + }, + { + "epoch": 8.203125, + "grad_norm": 0.0003100395551882684, + "learning_rate": 8.995479871350229e-05, + "loss": 1.054319909599144e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 525, + "tokens/total": 7926620, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 169726 + }, + { + "epoch": 8.21875, + "grad_norm": 0.008461945690214634, + "learning_rate": 8.990900133670319e-05, + "loss": 0.00010419739555800334, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0001, + "step": 526, + "tokens/total": 7941640, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 170048 + }, + { + "epoch": 8.234375, + "grad_norm": 0.0012957158032804728, + "learning_rate": 8.986311297875786e-05, + "loss": 2.4573982955189422e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 527, + "tokens/total": 7956708, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 170366 + }, + { + "epoch": 8.25, + "grad_norm": 0.0003723593254107982, + "learning_rate": 8.981713375926225e-05, + "loss": 1.1510109288792592e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 528, + "tokens/total": 7971900, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 170688 + }, + { + "epoch": 8.265625, + "grad_norm": 0.013197900727391243, + "learning_rate": 8.977106379804911e-05, + "loss": 3.1454932468477637e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 529, + "tokens/total": 7986976, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 171014 + }, + { + "epoch": 8.28125, + "grad_norm": 0.0002852350298780948, + "learning_rate": 8.97249032151877e-05, + "loss": 1.09092479760875e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 530, + "tokens/total": 8001928, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 171337 + }, + { + "epoch": 8.296875, + "grad_norm": 0.014826999045908451, + "learning_rate": 8.967865213098347e-05, + "loss": 0.00010147741704713553, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 531, + "tokens/total": 8017012, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 171662 + }, + { + "epoch": 8.3125, + "grad_norm": 0.5458815693855286, + "learning_rate": 8.963231066597771e-05, + "loss": 0.005835453514009714, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00585, + "step": 532, + "tokens/total": 8032012, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 171987 + }, + { + "epoch": 8.328125, + "grad_norm": 0.001101961126551032, + "learning_rate": 8.95858789409473e-05, + "loss": 1.618544411030598e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 533, + "tokens/total": 8047000, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 172313 + }, + { + "epoch": 8.34375, + "grad_norm": 0.0038681430742144585, + "learning_rate": 8.953935707690434e-05, + "loss": 3.059169102925807e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 534, + "tokens/total": 8062088, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 172637 + }, + { + "epoch": 8.359375, + "grad_norm": 0.00867560226470232, + "learning_rate": 8.949274519509585e-05, + "loss": 3.366108285263181e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00003, + "step": 535, + "tokens/total": 8077132, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 172961 + }, + { + "epoch": 8.375, + "grad_norm": 0.0022700580302625895, + "learning_rate": 8.944604341700346e-05, + "loss": 2.283003050251864e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 536, + "tokens/total": 8092276, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 173283 + }, + { + "epoch": 8.390625, + "grad_norm": 0.003660684684291482, + "learning_rate": 8.939925186434308e-05, + "loss": 5.109370977152139e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 537, + "tokens/total": 8107396, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 173603 + }, + { + "epoch": 8.40625, + "grad_norm": 0.005133552476763725, + "learning_rate": 8.93523706590646e-05, + "loss": 4.74297339678742e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00005, + "step": 538, + "tokens/total": 8122408, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 173926 + }, + { + "epoch": 8.421875, + "grad_norm": 0.2128029763698578, + "learning_rate": 8.930539992335164e-05, + "loss": 0.0008521364070475101, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00085, + "step": 539, + "tokens/total": 8137452, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 174247 + }, + { + "epoch": 8.4375, + "grad_norm": 1.6053714752197266, + "learning_rate": 8.9258339779621e-05, + "loss": 0.017414025962352753, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01757, + "step": 540, + "tokens/total": 8152540, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 174573 + }, + { + "epoch": 8.453125, + "grad_norm": 0.18711458146572113, + "learning_rate": 8.921119035052265e-05, + "loss": 0.0014177690027281642, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00142, + "step": 541, + "tokens/total": 8167692, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 174898 + }, + { + "epoch": 8.46875, + "grad_norm": 0.14626744389533997, + "learning_rate": 8.916395175893916e-05, + "loss": 0.001281649456359446, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00128, + "step": 542, + "tokens/total": 8182760, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 175225 + }, + { + "epoch": 8.484375, + "grad_norm": 0.0024803634732961655, + "learning_rate": 8.911662412798553e-05, + "loss": 3.1002535251900554e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00003, + "step": 543, + "tokens/total": 8197968, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 175550 + }, + { + "epoch": 8.5, + "grad_norm": 0.004511975217610598, + "learning_rate": 8.906920758100882e-05, + "loss": 7.237551471916959e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00007, + "step": 544, + "tokens/total": 8213064, + "tokens/train_per_sec_per_gpu": 10.04, + "tokens/trainable": 175872 + }, + { + "epoch": 8.515625, + "grad_norm": 0.0012622056528925896, + "learning_rate": 8.902170224158778e-05, + "loss": 1.5453677406185307e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 545, + "tokens/total": 8228148, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 176193 + }, + { + "epoch": 8.53125, + "grad_norm": 0.001151581876911223, + "learning_rate": 8.897410823353264e-05, + "loss": 1.9299024643260054e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 546, + "tokens/total": 8243252, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 176514 + }, + { + "epoch": 8.546875, + "grad_norm": 0.949654757976532, + "learning_rate": 8.892642568088465e-05, + "loss": 0.005113059654831886, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00513, + "step": 547, + "tokens/total": 8258448, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 176839 + }, + { + "epoch": 8.5625, + "grad_norm": 0.02893899753689766, + "learning_rate": 8.887865470791589e-05, + "loss": 9.800348925637081e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0001, + "step": 548, + "tokens/total": 8273528, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 177167 + }, + { + "epoch": 8.578125, + "grad_norm": 0.3177676498889923, + "learning_rate": 8.883079543912883e-05, + "loss": 0.0007184468558989465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00072, + "step": 549, + "tokens/total": 8288496, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 177491 + }, + { + "epoch": 8.59375, + "grad_norm": 0.09711069613695145, + "learning_rate": 8.878284799925612e-05, + "loss": 0.0003404346061870456, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00034, + "step": 550, + "tokens/total": 8303696, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 177818 + }, + { + "epoch": 8.609375, + "grad_norm": 0.4529828131198883, + "learning_rate": 8.873481251326016e-05, + "loss": 0.0012388179311528802, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00124, + "step": 551, + "tokens/total": 8318700, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 178144 + }, + { + "epoch": 8.625, + "grad_norm": 0.03608764708042145, + "learning_rate": 8.868668910633283e-05, + "loss": 0.00011354131856933236, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 552, + "tokens/total": 8333792, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 178466 + }, + { + "epoch": 8.640625, + "grad_norm": 0.003121917136013508, + "learning_rate": 8.863847790389513e-05, + "loss": 2.8653050321736373e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 553, + "tokens/total": 8348860, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 178793 + }, + { + "epoch": 8.65625, + "grad_norm": 0.06226268410682678, + "learning_rate": 8.859017903159695e-05, + "loss": 0.00021325259876903147, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00021, + "step": 554, + "tokens/total": 8364084, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 179113 + }, + { + "epoch": 8.671875, + "grad_norm": 2.6193294525146484, + "learning_rate": 8.854179261531659e-05, + "loss": 0.013566155917942524, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.01366, + "step": 555, + "tokens/total": 8379180, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 179436 + }, + { + "epoch": 8.6875, + "grad_norm": 0.3264051079750061, + "learning_rate": 8.849331878116055e-05, + "loss": 0.002084167441353202, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00209, + "step": 556, + "tokens/total": 8394380, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 179753 + }, + { + "epoch": 8.703125, + "grad_norm": 0.0010522427037358284, + "learning_rate": 8.844475765546314e-05, + "loss": 1.890875500976108e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.00002, + "step": 557, + "tokens/total": 8409332, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 180074 + }, + { + "epoch": 8.71875, + "grad_norm": 0.05134303867816925, + "learning_rate": 8.83961093647862e-05, + "loss": 0.00012807887105736881, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00013, + "step": 558, + "tokens/total": 8424388, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 180403 + }, + { + "epoch": 8.734375, + "grad_norm": 1.0385314226150513, + "learning_rate": 8.834737403591869e-05, + "loss": 0.02681276947259903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02718, + "step": 559, + "tokens/total": 8439436, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 180730 + }, + { + "epoch": 8.75, + "grad_norm": 0.002918509067967534, + "learning_rate": 8.82985517958765e-05, + "loss": 3.538604505592957e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 560, + "tokens/total": 8454568, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 181051 + }, + { + "epoch": 8.765625, + "grad_norm": 0.3665774464607239, + "learning_rate": 8.824964277190194e-05, + "loss": 0.0011759663466364145, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00118, + "step": 561, + "tokens/total": 8469748, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 181374 + }, + { + "epoch": 8.78125, + "grad_norm": 0.003580427495762706, + "learning_rate": 8.820064709146357e-05, + "loss": 4.611169424606487e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 562, + "tokens/total": 8484772, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 181698 + }, + { + "epoch": 8.796875, + "grad_norm": 0.06990180164575577, + "learning_rate": 8.815156488225575e-05, + "loss": 0.0006263494142331183, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00063, + "step": 563, + "tokens/total": 8499748, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 182020 + }, + { + "epoch": 8.8125, + "grad_norm": 0.0055811540223658085, + "learning_rate": 8.810239627219837e-05, + "loss": 8.284625073429197e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 564, + "tokens/total": 8514840, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 182344 + }, + { + "epoch": 8.828125, + "grad_norm": 0.35078662633895874, + "learning_rate": 8.805314138943649e-05, + "loss": 0.0007666668388992548, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00077, + "step": 565, + "tokens/total": 8529904, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 182664 + }, + { + "epoch": 8.84375, + "grad_norm": 0.0014952566707506776, + "learning_rate": 8.800380036234005e-05, + "loss": 4.0127793909050524e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 566, + "tokens/total": 8545076, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 182984 + }, + { + "epoch": 8.859375, + "grad_norm": 8.584396362304688, + "learning_rate": 8.795437331950349e-05, + "loss": 0.00995005201548338, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01, + "step": 567, + "tokens/total": 8560164, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 183306 + }, + { + "epoch": 8.875, + "grad_norm": 0.06884938478469849, + "learning_rate": 8.790486038974539e-05, + "loss": 0.00021491909865289927, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00021, + "step": 568, + "tokens/total": 8575364, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 183630 + }, + { + "epoch": 8.890625, + "grad_norm": 0.4572925865650177, + "learning_rate": 8.785526170210823e-05, + "loss": 0.03290176764130592, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03345, + "step": 569, + "tokens/total": 8590504, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 183950 + }, + { + "epoch": 8.90625, + "grad_norm": 0.0031276054214686155, + "learning_rate": 8.780557738585795e-05, + "loss": 7.353207911364734e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00007, + "step": 570, + "tokens/total": 8605536, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 184273 + }, + { + "epoch": 8.921875, + "grad_norm": 0.005127928219735622, + "learning_rate": 8.775580757048369e-05, + "loss": 0.00018162198830395937, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00018, + "step": 571, + "tokens/total": 8620496, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 184593 + }, + { + "epoch": 8.9375, + "grad_norm": 0.4080176055431366, + "learning_rate": 8.770595238569737e-05, + "loss": 0.0027535860426723957, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00276, + "step": 572, + "tokens/total": 8635532, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 184917 + }, + { + "epoch": 8.953125, + "grad_norm": 0.2424798607826233, + "learning_rate": 8.765601196143347e-05, + "loss": 0.007297150790691376, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00732, + "step": 573, + "tokens/total": 8650684, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 185241 + }, + { + "epoch": 8.96875, + "grad_norm": 0.11064363270998001, + "learning_rate": 8.76059864278486e-05, + "loss": 0.0007703031878918409, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00077, + "step": 574, + "tokens/total": 8665752, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 185565 + }, + { + "epoch": 8.984375, + "grad_norm": 0.12166444212198257, + "learning_rate": 8.755587591532116e-05, + "loss": 0.001453407690860331, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00145, + "step": 575, + "tokens/total": 8680856, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 185886 + }, + { + "epoch": 9.0, + "grad_norm": 0.01695314794778824, + "learning_rate": 8.750568055445105e-05, + "loss": 0.00043913544504903257, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00044, + "step": 576, + "tokens/total": 8695996, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 186210 + }, + { + "epoch": 9.015625, + "grad_norm": 0.03284699469804764, + "learning_rate": 8.74554004760593e-05, + "loss": 0.0009203513618558645, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00092, + "step": 577, + "tokens/total": 8711112, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 186533 + }, + { + "epoch": 9.03125, + "grad_norm": 0.2037004828453064, + "learning_rate": 8.740503581118774e-05, + "loss": 0.0039493246003985405, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00396, + "step": 578, + "tokens/total": 8726132, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 186855 + }, + { + "epoch": 9.046875, + "grad_norm": 0.015510317869484425, + "learning_rate": 8.735458669109863e-05, + "loss": 0.00044698762940242887, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00045, + "step": 579, + "tokens/total": 8741288, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 187179 + }, + { + "epoch": 9.0625, + "grad_norm": 0.04965129867196083, + "learning_rate": 8.730405324727435e-05, + "loss": 0.000823804410174489, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00082, + "step": 580, + "tokens/total": 8756480, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 187503 + }, + { + "epoch": 9.078125, + "grad_norm": 0.006426906678825617, + "learning_rate": 8.72534356114171e-05, + "loss": 0.00018337769142817706, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00018, + "step": 581, + "tokens/total": 8771664, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 187824 + }, + { + "epoch": 9.09375, + "grad_norm": 0.007196735590696335, + "learning_rate": 8.720273391544839e-05, + "loss": 0.00023079977836459875, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00023, + "step": 582, + "tokens/total": 8786788, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 188147 + }, + { + "epoch": 9.109375, + "grad_norm": 0.30709800124168396, + "learning_rate": 8.715194829150892e-05, + "loss": 0.0022993457969278097, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0023, + "step": 583, + "tokens/total": 8801964, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 188471 + }, + { + "epoch": 9.125, + "grad_norm": 0.01921996846795082, + "learning_rate": 8.710107887195807e-05, + "loss": 0.00013834233686793596, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 584, + "tokens/total": 8817248, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 188799 + }, + { + "epoch": 9.140625, + "grad_norm": 0.038318585604429245, + "learning_rate": 8.705012578937364e-05, + "loss": 0.00045493399375118315, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00046, + "step": 585, + "tokens/total": 8832416, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 189119 + }, + { + "epoch": 9.15625, + "grad_norm": 0.44059014320373535, + "learning_rate": 8.699908917655145e-05, + "loss": 0.0013400256866589189, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00134, + "step": 586, + "tokens/total": 8847412, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 189437 + }, + { + "epoch": 9.171875, + "grad_norm": 0.0677911713719368, + "learning_rate": 8.694796916650503e-05, + "loss": 0.00023939728271216154, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00024, + "step": 587, + "tokens/total": 8862508, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 189761 + }, + { + "epoch": 9.1875, + "grad_norm": 0.0946345254778862, + "learning_rate": 8.689676589246528e-05, + "loss": 0.0004921150393784046, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00049, + "step": 588, + "tokens/total": 8877616, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 190085 + }, + { + "epoch": 9.203125, + "grad_norm": 0.6251227259635925, + "learning_rate": 8.684547948788008e-05, + "loss": 0.005682742223143578, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0057, + "step": 589, + "tokens/total": 8892752, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 190410 + }, + { + "epoch": 9.21875, + "grad_norm": 0.0019117683405056596, + "learning_rate": 8.679411008641397e-05, + "loss": 5.5858457926660776e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 590, + "tokens/total": 8907852, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 190732 + }, + { + "epoch": 9.234375, + "grad_norm": 0.0006620520143769681, + "learning_rate": 8.674265782194781e-05, + "loss": 2.6435986001160927e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00003, + "step": 591, + "tokens/total": 8922884, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 191054 + }, + { + "epoch": 9.25, + "grad_norm": 0.0028546350076794624, + "learning_rate": 8.66911228285784e-05, + "loss": 3.963516064686701e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00004, + "step": 592, + "tokens/total": 8937984, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 191383 + }, + { + "epoch": 9.265625, + "grad_norm": 0.0007748997304588556, + "learning_rate": 8.663950524061822e-05, + "loss": 2.5786108380998485e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 593, + "tokens/total": 8953168, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 191709 + }, + { + "epoch": 9.28125, + "grad_norm": 0.0005234112613834441, + "learning_rate": 8.658780519259493e-05, + "loss": 2.25883413804695e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 594, + "tokens/total": 8968100, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 192030 + }, + { + "epoch": 9.296875, + "grad_norm": 0.0004649423935916275, + "learning_rate": 8.653602281925113e-05, + "loss": 2.2441228793468326e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 595, + "tokens/total": 8983324, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 192352 + }, + { + "epoch": 9.3125, + "grad_norm": 0.08911292999982834, + "learning_rate": 8.648415825554399e-05, + "loss": 0.0005657169385813177, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00057, + "step": 596, + "tokens/total": 8998508, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 192675 + }, + { + "epoch": 9.328125, + "grad_norm": 0.0026829498820006847, + "learning_rate": 8.643221163664483e-05, + "loss": 3.790732444031164e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 597, + "tokens/total": 9013568, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 192995 + }, + { + "epoch": 9.34375, + "grad_norm": 0.0018017059192061424, + "learning_rate": 8.638018309793893e-05, + "loss": 2.4644470613566227e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 598, + "tokens/total": 9028760, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 193317 + }, + { + "epoch": 9.359375, + "grad_norm": 0.0754798874258995, + "learning_rate": 8.632807277502501e-05, + "loss": 0.00029164052102714777, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00029, + "step": 599, + "tokens/total": 9043940, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 193644 + }, + { + "epoch": 9.375, + "grad_norm": 0.00033408598392270505, + "learning_rate": 8.627588080371493e-05, + "loss": 1.5072258065629285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 600, + "tokens/total": 9059108, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 193969 + }, + { + "epoch": 9.390625, + "grad_norm": 0.00043498555896803737, + "learning_rate": 8.622360732003337e-05, + "loss": 1.83389875019202e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 601, + "tokens/total": 9074212, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 194291 + }, + { + "epoch": 9.40625, + "grad_norm": 0.0006206405232660472, + "learning_rate": 8.617125246021742e-05, + "loss": 2.0851886802120134e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 602, + "tokens/total": 9089180, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 194612 + }, + { + "epoch": 9.421875, + "grad_norm": 0.00028540706261992455, + "learning_rate": 8.611881636071627e-05, + "loss": 1.3558287719206419e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 603, + "tokens/total": 9104272, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 194933 + }, + { + "epoch": 9.4375, + "grad_norm": 0.0004516422050073743, + "learning_rate": 8.60662991581909e-05, + "loss": 1.6650086763547733e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 604, + "tokens/total": 9119376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 195252 + }, + { + "epoch": 9.453125, + "grad_norm": 0.002324763685464859, + "learning_rate": 8.601370098951355e-05, + "loss": 3.069010563194752e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 605, + "tokens/total": 9134396, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 195574 + }, + { + "epoch": 9.46875, + "grad_norm": 0.00039453315548598766, + "learning_rate": 8.596102199176755e-05, + "loss": 1.3730700629821513e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 606, + "tokens/total": 9149608, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 195900 + }, + { + "epoch": 9.484375, + "grad_norm": 0.0003811897477135062, + "learning_rate": 8.59082623022469e-05, + "loss": 1.661576607148163e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00002, + "step": 607, + "tokens/total": 9164832, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 196225 + }, + { + "epoch": 9.5, + "grad_norm": 0.00027696648612618446, + "learning_rate": 8.585542205845588e-05, + "loss": 1.0936594662780408e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 608, + "tokens/total": 9179884, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 196550 + }, + { + "epoch": 9.515625, + "grad_norm": 0.00039244198706001043, + "learning_rate": 8.580250139810868e-05, + "loss": 1.2854216038249433e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 609, + "tokens/total": 9194960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 196874 + }, + { + "epoch": 9.53125, + "grad_norm": 0.00029694641125388443, + "learning_rate": 8.57495004591291e-05, + "loss": 1.0953734090435319e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 610, + "tokens/total": 9210108, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 197195 + }, + { + "epoch": 9.546875, + "grad_norm": 0.0006175542366690934, + "learning_rate": 8.56964193796502e-05, + "loss": 1.0757680684037041e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 611, + "tokens/total": 9225252, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 197519 + }, + { + "epoch": 9.5625, + "grad_norm": 0.00114785460755229, + "learning_rate": 8.56432582980139e-05, + "loss": 1.7286893125856295e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 612, + "tokens/total": 9240320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 197843 + }, + { + "epoch": 9.578125, + "grad_norm": 0.0005527856992557645, + "learning_rate": 8.559001735277055e-05, + "loss": 1.5963505575200543e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 613, + "tokens/total": 9255456, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 198165 + }, + { + "epoch": 9.59375, + "grad_norm": 0.00021238153567537665, + "learning_rate": 8.553669668267871e-05, + "loss": 7.194825684564421e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 614, + "tokens/total": 9270556, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 198486 + }, + { + "epoch": 9.609375, + "grad_norm": 0.0012781359255313873, + "learning_rate": 8.548329642670472e-05, + "loss": 2.0431305529200472e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00002, + "step": 615, + "tokens/total": 9285660, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 198814 + }, + { + "epoch": 9.625, + "grad_norm": 0.00020792640862055123, + "learning_rate": 8.542981672402233e-05, + "loss": 7.524381999246543e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 616, + "tokens/total": 9300668, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 199136 + }, + { + "epoch": 9.640625, + "grad_norm": 0.0003108520177192986, + "learning_rate": 8.537625771401231e-05, + "loss": 1.0943025699816644e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 617, + "tokens/total": 9315668, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 199460 + }, + { + "epoch": 9.65625, + "grad_norm": 0.0002239548193756491, + "learning_rate": 8.53226195362622e-05, + "loss": 9.883589882520027e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 618, + "tokens/total": 9330848, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 199788 + }, + { + "epoch": 9.671875, + "grad_norm": 0.00030563052860088646, + "learning_rate": 8.526890233056579e-05, + "loss": 9.192094694299158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 619, + "tokens/total": 9345924, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 200110 + }, + { + "epoch": 9.6875, + "grad_norm": 0.0312703438103199, + "learning_rate": 8.521510623692288e-05, + "loss": 0.00012628451804630458, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 620, + "tokens/total": 9361068, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 200438 + }, + { + "epoch": 9.703125, + "grad_norm": 0.00016913052240852267, + "learning_rate": 8.516123139553889e-05, + "loss": 6.865585874038516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 621, + "tokens/total": 9376068, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 200762 + }, + { + "epoch": 9.71875, + "grad_norm": 0.00023767122183926404, + "learning_rate": 8.510727794682443e-05, + "loss": 9.91942852124339e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 622, + "tokens/total": 9391068, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 201087 + }, + { + "epoch": 9.734375, + "grad_norm": 0.0003941507893614471, + "learning_rate": 8.505324603139499e-05, + "loss": 1.4861869203741662e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 623, + "tokens/total": 9406204, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 201407 + }, + { + "epoch": 9.75, + "grad_norm": 0.00018824409926310182, + "learning_rate": 8.499913579007058e-05, + "loss": 8.226126738009043e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 624, + "tokens/total": 9421308, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 201727 + }, + { + "epoch": 9.765625, + "grad_norm": 0.0001939542853506282, + "learning_rate": 8.494494736387535e-05, + "loss": 8.253552550741006e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 625, + "tokens/total": 9436284, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 202052 + }, + { + "epoch": 9.78125, + "grad_norm": 0.0003432348894421011, + "learning_rate": 8.489068089403719e-05, + "loss": 1.0490179192856885e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 626, + "tokens/total": 9451392, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 202375 + }, + { + "epoch": 9.796875, + "grad_norm": 0.0004783475014846772, + "learning_rate": 8.483633652198739e-05, + "loss": 1.068074743670877e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 627, + "tokens/total": 9466440, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 202706 + }, + { + "epoch": 9.8125, + "grad_norm": 0.00028024587663821876, + "learning_rate": 8.47819143893603e-05, + "loss": 9.895077710098121e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 628, + "tokens/total": 9481532, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 203028 + }, + { + "epoch": 9.828125, + "grad_norm": 0.00017581306747160852, + "learning_rate": 8.472741463799296e-05, + "loss": 6.723947535647312e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 629, + "tokens/total": 9496764, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 203351 + }, + { + "epoch": 9.84375, + "grad_norm": 0.00013647547166328877, + "learning_rate": 8.467283740992458e-05, + "loss": 5.100245743960841e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 630, + "tokens/total": 9511892, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 203673 + }, + { + "epoch": 9.859375, + "grad_norm": 0.00022224783606361598, + "learning_rate": 8.461818284739645e-05, + "loss": 7.700567948631942e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.00001, + "step": 631, + "tokens/total": 9526912, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 203992 + }, + { + "epoch": 9.875, + "grad_norm": 0.033552926033735275, + "learning_rate": 8.456345109285131e-05, + "loss": 7.867354725021869e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00008, + "step": 632, + "tokens/total": 9542048, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 204316 + }, + { + "epoch": 9.890625, + "grad_norm": 0.00016879606118891388, + "learning_rate": 8.45086422889331e-05, + "loss": 6.822014256613329e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 633, + "tokens/total": 9557160, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 204640 + }, + { + "epoch": 9.90625, + "grad_norm": 0.0007644848665222526, + "learning_rate": 8.445375657848657e-05, + "loss": 1.4742925486643799e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 634, + "tokens/total": 9572248, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 204962 + }, + { + "epoch": 9.921875, + "grad_norm": 0.00015245663234964013, + "learning_rate": 8.439879410455693e-05, + "loss": 6.433082944568014e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 635, + "tokens/total": 9587332, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 205286 + }, + { + "epoch": 9.9375, + "grad_norm": 0.00024901528377085924, + "learning_rate": 8.434375501038944e-05, + "loss": 7.426990123349242e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 636, + "tokens/total": 9602432, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 205611 + }, + { + "epoch": 9.953125, + "grad_norm": 0.00020785727247130126, + "learning_rate": 8.428863943942904e-05, + "loss": 4.663966592488578e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 637, + "tokens/total": 9617432, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 205935 + }, + { + "epoch": 9.96875, + "grad_norm": 0.0003147262614220381, + "learning_rate": 8.423344753531999e-05, + "loss": 1.0147239663638175e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 638, + "tokens/total": 9632656, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 206254 + }, + { + "epoch": 9.984375, + "grad_norm": 0.00011762624490074813, + "learning_rate": 8.417817944190553e-05, + "loss": 5.135711944603827e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 639, + "tokens/total": 9647680, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 206574 + }, + { + "epoch": 10.0, + "grad_norm": 0.00015940859157126397, + "learning_rate": 8.412283530322736e-05, + "loss": 6.558111181220738e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 640, + "tokens/total": 9662672, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 206900 + }, + { + "epoch": 10.015625, + "grad_norm": 0.00020247366046532989, + "learning_rate": 8.406741526352553e-05, + "loss": 7.633244422322605e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 641, + "tokens/total": 9677748, + "tokens/train_per_sec_per_gpu": 8.85, + "tokens/trainable": 207225 + }, + { + "epoch": 10.03125, + "grad_norm": 0.0004867524839937687, + "learning_rate": 8.401191946723776e-05, + "loss": 1.1116100722574629e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 642, + "tokens/total": 9692824, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 207549 + }, + { + "epoch": 10.046875, + "grad_norm": 0.00011772097059292719, + "learning_rate": 8.395634805899931e-05, + "loss": 4.867678399023134e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 643, + "tokens/total": 9707908, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 207868 + }, + { + "epoch": 10.0625, + "grad_norm": 0.00015480454021599144, + "learning_rate": 8.390070118364246e-05, + "loss": 5.00797386848717e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 644, + "tokens/total": 9722956, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 208190 + }, + { + "epoch": 10.078125, + "grad_norm": 0.00012757439981214702, + "learning_rate": 8.384497898619617e-05, + "loss": 5.3190769904176705e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 645, + "tokens/total": 9738088, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 208514 + }, + { + "epoch": 10.09375, + "grad_norm": 0.0001416898739989847, + "learning_rate": 8.378918161188571e-05, + "loss": 5.836336185893742e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 646, + "tokens/total": 9753164, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 208839 + }, + { + "epoch": 10.109375, + "grad_norm": 0.00010145518172066659, + "learning_rate": 8.373330920613229e-05, + "loss": 4.150137101532891e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 647, + "tokens/total": 9768176, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 209161 + }, + { + "epoch": 10.125, + "grad_norm": 0.0002767986443359405, + "learning_rate": 8.367736191455268e-05, + "loss": 6.726240826537833e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 648, + "tokens/total": 9783276, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 209483 + }, + { + "epoch": 10.140625, + "grad_norm": 0.00012657605111598969, + "learning_rate": 8.362133988295879e-05, + "loss": 5.29480712430086e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 649, + "tokens/total": 9798340, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 209810 + }, + { + "epoch": 10.15625, + "grad_norm": 0.00016513089940417558, + "learning_rate": 8.356524325735735e-05, + "loss": 4.137089945288608e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 650, + "tokens/total": 9813404, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 210132 + }, + { + "epoch": 10.171875, + "grad_norm": 0.00013624246639665216, + "learning_rate": 8.35090721839495e-05, + "loss": 5.879107447981369e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 651, + "tokens/total": 9828520, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 210459 + }, + { + "epoch": 10.1875, + "grad_norm": 0.00032070872839540243, + "learning_rate": 8.345282680913033e-05, + "loss": 7.33726710677729e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 652, + "tokens/total": 9843532, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 210785 + }, + { + "epoch": 10.203125, + "grad_norm": 0.00013062500511296093, + "learning_rate": 8.33965072794887e-05, + "loss": 5.269500888971379e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 653, + "tokens/total": 9858592, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 211109 + }, + { + "epoch": 10.21875, + "grad_norm": 0.000225347641389817, + "learning_rate": 8.334011374180668e-05, + "loss": 6.59221041132696e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 654, + "tokens/total": 9873660, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 211432 + }, + { + "epoch": 10.234375, + "grad_norm": 0.00013657586532644928, + "learning_rate": 8.328364634305917e-05, + "loss": 5.456272447190713e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 655, + "tokens/total": 9888736, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 211754 + }, + { + "epoch": 10.25, + "grad_norm": 3.6042065620422363, + "learning_rate": 8.322710523041366e-05, + "loss": 0.014105875976383686, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01421, + "step": 656, + "tokens/total": 9903756, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 212075 + }, + { + "epoch": 10.265625, + "grad_norm": 0.00015812524361535907, + "learning_rate": 8.317049055122971e-05, + "loss": 5.314343525242293e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 657, + "tokens/total": 9918800, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 212399 + }, + { + "epoch": 10.28125, + "grad_norm": 0.00023147545289248228, + "learning_rate": 8.311380245305863e-05, + "loss": 6.949136150069535e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 658, + "tokens/total": 9933768, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 212718 + }, + { + "epoch": 10.296875, + "grad_norm": 0.00014753777941223234, + "learning_rate": 8.305704108364301e-05, + "loss": 5.799164227937581e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 659, + "tokens/total": 9948864, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 213040 + }, + { + "epoch": 10.3125, + "grad_norm": 0.0011448130244389176, + "learning_rate": 8.30002065909165e-05, + "loss": 2.2385131160262972e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 660, + "tokens/total": 9964024, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 213361 + }, + { + "epoch": 10.328125, + "grad_norm": 0.11430440843105316, + "learning_rate": 8.294329912300329e-05, + "loss": 0.0007114034378901124, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00071, + "step": 661, + "tokens/total": 9979200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 213682 + }, + { + "epoch": 10.34375, + "grad_norm": 1.3747159242630005, + "learning_rate": 8.288631882821773e-05, + "loss": 0.007618979085236788, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00765, + "step": 662, + "tokens/total": 9994184, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 214002 + }, + { + "epoch": 10.359375, + "grad_norm": 0.0006237569032236934, + "learning_rate": 8.282926585506399e-05, + "loss": 9.479489563091192e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 663, + "tokens/total": 10009244, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 214318 + }, + { + "epoch": 10.375, + "grad_norm": 0.0012571830302476883, + "learning_rate": 8.277214035223568e-05, + "loss": 1.839931974245701e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 664, + "tokens/total": 10024264, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 214647 + }, + { + "epoch": 10.390625, + "grad_norm": 0.33910050988197327, + "learning_rate": 8.271494246861545e-05, + "loss": 0.0007400690228678286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00074, + "step": 665, + "tokens/total": 10039468, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 214968 + }, + { + "epoch": 10.40625, + "grad_norm": 0.0007855923031456769, + "learning_rate": 8.26576723532745e-05, + "loss": 1.8088217984768562e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 666, + "tokens/total": 10054640, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 215291 + }, + { + "epoch": 10.421875, + "grad_norm": 0.17555910348892212, + "learning_rate": 8.260033015547241e-05, + "loss": 0.0004138879885431379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00041, + "step": 667, + "tokens/total": 10069820, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 215612 + }, + { + "epoch": 10.4375, + "grad_norm": 0.0006887784111313522, + "learning_rate": 8.254291602465653e-05, + "loss": 1.740579136821907e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 668, + "tokens/total": 10084928, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 215940 + }, + { + "epoch": 10.453125, + "grad_norm": 0.0003940793394576758, + "learning_rate": 8.248543011046174e-05, + "loss": 1.1545018423930742e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 669, + "tokens/total": 10099972, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 216264 + }, + { + "epoch": 10.46875, + "grad_norm": 0.0041761621832847595, + "learning_rate": 8.242787256270995e-05, + "loss": 4.01059805881232e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00004, + "step": 670, + "tokens/total": 10115120, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 216588 + }, + { + "epoch": 10.484375, + "grad_norm": 0.006432351656258106, + "learning_rate": 8.237024353140983e-05, + "loss": 4.4539981900015846e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 671, + "tokens/total": 10130164, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 216911 + }, + { + "epoch": 10.5, + "grad_norm": 0.008654266595840454, + "learning_rate": 8.231254316675629e-05, + "loss": 9.718702494865283e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 672, + "tokens/total": 10145220, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 217239 + }, + { + "epoch": 10.515625, + "grad_norm": 0.0033223514910787344, + "learning_rate": 8.22547716191302e-05, + "loss": 2.6446699848747812e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00003, + "step": 673, + "tokens/total": 10160416, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 217566 + }, + { + "epoch": 10.53125, + "grad_norm": 0.0029003892559558153, + "learning_rate": 8.219692903909794e-05, + "loss": 4.9929614760912955e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00005, + "step": 674, + "tokens/total": 10175636, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 217890 + }, + { + "epoch": 10.546875, + "grad_norm": 0.0760011374950409, + "learning_rate": 8.213901557741098e-05, + "loss": 0.0007359297596849501, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00074, + "step": 675, + "tokens/total": 10190752, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 218218 + }, + { + "epoch": 10.5625, + "grad_norm": 0.2406068593263626, + "learning_rate": 8.208103138500557e-05, + "loss": 0.0019148560240864754, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00192, + "step": 676, + "tokens/total": 10205880, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 218539 + }, + { + "epoch": 10.578125, + "grad_norm": 0.0008238900336436927, + "learning_rate": 8.202297661300228e-05, + "loss": 1.983395304705482e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 677, + "tokens/total": 10220984, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 218861 + }, + { + "epoch": 10.59375, + "grad_norm": 0.0006221993826329708, + "learning_rate": 8.196485141270563e-05, + "loss": 1.3678405593964271e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 678, + "tokens/total": 10236212, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 219186 + }, + { + "epoch": 10.609375, + "grad_norm": 0.0014093968784436584, + "learning_rate": 8.190665593560367e-05, + "loss": 1.4763102626602631e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 679, + "tokens/total": 10251296, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 219510 + }, + { + "epoch": 10.625, + "grad_norm": 0.4953056871891022, + "learning_rate": 8.184839033336767e-05, + "loss": 0.015656938776373863, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01578, + "step": 680, + "tokens/total": 10266344, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 219834 + }, + { + "epoch": 10.640625, + "grad_norm": 0.0181986466050148, + "learning_rate": 8.179005475785162e-05, + "loss": 0.00013063072401564568, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00013, + "step": 681, + "tokens/total": 10281380, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 220156 + }, + { + "epoch": 10.65625, + "grad_norm": 0.008455532602965832, + "learning_rate": 8.173164936109184e-05, + "loss": 0.00016575312474742532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00017, + "step": 682, + "tokens/total": 10296504, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 220479 + }, + { + "epoch": 10.671875, + "grad_norm": 0.0012320041423663497, + "learning_rate": 8.167317429530669e-05, + "loss": 3.3237243769690394e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00003, + "step": 683, + "tokens/total": 10311544, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 220803 + }, + { + "epoch": 10.6875, + "grad_norm": 0.00673325639218092, + "learning_rate": 8.161462971289608e-05, + "loss": 0.00015895556134637445, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00016, + "step": 684, + "tokens/total": 10326644, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 221124 + }, + { + "epoch": 10.703125, + "grad_norm": 0.1733241081237793, + "learning_rate": 8.155601576644108e-05, + "loss": 0.0025445870123803616, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00255, + "step": 685, + "tokens/total": 10341824, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 221449 + }, + { + "epoch": 10.71875, + "grad_norm": 0.021975671872496605, + "learning_rate": 8.14973326087036e-05, + "loss": 0.0004545206611510366, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00045, + "step": 686, + "tokens/total": 10357076, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 221774 + }, + { + "epoch": 10.734375, + "grad_norm": 0.21455904841423035, + "learning_rate": 8.14385803926258e-05, + "loss": 0.004639237187802792, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00465, + "step": 687, + "tokens/total": 10372192, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 222094 + }, + { + "epoch": 10.75, + "grad_norm": 0.028662165626883507, + "learning_rate": 8.137975927132995e-05, + "loss": 0.0003142151399515569, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 688, + "tokens/total": 10387336, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 222419 + }, + { + "epoch": 10.765625, + "grad_norm": 0.0500834695994854, + "learning_rate": 8.132086939811787e-05, + "loss": 0.0010720632271841168, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00107, + "step": 689, + "tokens/total": 10402484, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 222742 + }, + { + "epoch": 10.78125, + "grad_norm": 0.04645442217588425, + "learning_rate": 8.126191092647053e-05, + "loss": 0.00026277147117070854, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00026, + "step": 690, + "tokens/total": 10417640, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 223062 + }, + { + "epoch": 10.796875, + "grad_norm": 0.007354910485446453, + "learning_rate": 8.120288401004771e-05, + "loss": 0.00011086190352216363, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00011, + "step": 691, + "tokens/total": 10432700, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 223383 + }, + { + "epoch": 10.8125, + "grad_norm": 0.00775782810524106, + "learning_rate": 8.114378880268758e-05, + "loss": 0.0001411090634064749, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 692, + "tokens/total": 10447872, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 223707 + }, + { + "epoch": 10.828125, + "grad_norm": 0.0007672952488064766, + "learning_rate": 8.108462545840627e-05, + "loss": 2.438546289340593e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 693, + "tokens/total": 10462960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 224031 + }, + { + "epoch": 10.84375, + "grad_norm": 0.03337325155735016, + "learning_rate": 8.102539413139751e-05, + "loss": 0.0002755741588771343, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00028, + "step": 694, + "tokens/total": 10477964, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 224352 + }, + { + "epoch": 10.859375, + "grad_norm": 0.008774218149483204, + "learning_rate": 8.09660949760322e-05, + "loss": 7.373647531494498e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00007, + "step": 695, + "tokens/total": 10493088, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 224677 + }, + { + "epoch": 10.875, + "grad_norm": 0.19777068495750427, + "learning_rate": 8.090672814685802e-05, + "loss": 0.002209917875006795, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00221, + "step": 696, + "tokens/total": 10508160, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 225001 + }, + { + "epoch": 10.890625, + "grad_norm": 0.0007365435594692826, + "learning_rate": 8.084729379859902e-05, + "loss": 2.1141760953469202e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 697, + "tokens/total": 10523108, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 225323 + }, + { + "epoch": 10.90625, + "grad_norm": 0.0004815941210836172, + "learning_rate": 8.078779208615524e-05, + "loss": 1.7367412510793656e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 698, + "tokens/total": 10538160, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 225645 + }, + { + "epoch": 10.921875, + "grad_norm": 0.00034571634023450315, + "learning_rate": 8.072822316460223e-05, + "loss": 1.359562884317711e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 699, + "tokens/total": 10553296, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 225974 + }, + { + "epoch": 10.9375, + "grad_norm": 0.04475097730755806, + "learning_rate": 8.066858718919078e-05, + "loss": 0.0004733581154141575, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00047, + "step": 700, + "tokens/total": 10568452, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 226296 + }, + { + "epoch": 10.953125, + "grad_norm": 0.00406939210370183, + "learning_rate": 8.060888431534638e-05, + "loss": 4.0564369555795565e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 701, + "tokens/total": 10583552, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 226619 + }, + { + "epoch": 10.96875, + "grad_norm": 0.0006428882479667664, + "learning_rate": 8.054911469866894e-05, + "loss": 1.8578755771159194e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 702, + "tokens/total": 10598668, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 226944 + }, + { + "epoch": 10.984375, + "grad_norm": 0.0004231981001794338, + "learning_rate": 8.048927849493222e-05, + "loss": 1.3509456039173529e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00001, + "step": 703, + "tokens/total": 10613828, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 227269 + }, + { + "epoch": 11.0, + "grad_norm": 0.0003220014041289687, + "learning_rate": 8.042937586008359e-05, + "loss": 1.1975650522799697e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 704, + "tokens/total": 10628960, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 227590 + }, + { + "epoch": 11.015625, + "grad_norm": 0.0004653835785575211, + "learning_rate": 8.036940695024354e-05, + "loss": 1.2614174011105206e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 705, + "tokens/total": 10644044, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 227915 + }, + { + "epoch": 11.03125, + "grad_norm": 0.0010091683361679316, + "learning_rate": 8.030937192170529e-05, + "loss": 2.3527491066488437e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 706, + "tokens/total": 10659112, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 228237 + }, + { + "epoch": 11.046875, + "grad_norm": 0.0002630383532959968, + "learning_rate": 8.02492709309344e-05, + "loss": 1.0589868907118216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 707, + "tokens/total": 10674172, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 228567 + }, + { + "epoch": 11.0625, + "grad_norm": 0.0003388511249795556, + "learning_rate": 8.018910413456828e-05, + "loss": 9.929903171723709e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 708, + "tokens/total": 10689384, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 228891 + }, + { + "epoch": 11.078125, + "grad_norm": 0.00021411596389953047, + "learning_rate": 8.012887168941592e-05, + "loss": 9.911593224387616e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 709, + "tokens/total": 10704556, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 229221 + }, + { + "epoch": 11.09375, + "grad_norm": 0.00035079781082458794, + "learning_rate": 8.006857375245736e-05, + "loss": 1.117552073992556e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 710, + "tokens/total": 10719640, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 229540 + }, + { + "epoch": 11.109375, + "grad_norm": 0.00019558356143534184, + "learning_rate": 8.000821048084334e-05, + "loss": 8.218433322326746e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 711, + "tokens/total": 10734776, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 229861 + }, + { + "epoch": 11.125, + "grad_norm": 0.0003149871772620827, + "learning_rate": 7.994778203189489e-05, + "loss": 8.92526895768242e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 712, + "tokens/total": 10749824, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 230186 + }, + { + "epoch": 11.140625, + "grad_norm": 0.00013192363257985562, + "learning_rate": 7.988728856310289e-05, + "loss": 4.4209923544258345e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 713, + "tokens/total": 10764920, + "tokens/train_per_sec_per_gpu": 8.44, + "tokens/trainable": 230511 + }, + { + "epoch": 11.15625, + "grad_norm": 0.004311064258217812, + "learning_rate": 7.982673023212767e-05, + "loss": 3.988208118244074e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00004, + "step": 714, + "tokens/total": 10780068, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 230834 + }, + { + "epoch": 11.171875, + "grad_norm": 0.0002725148806348443, + "learning_rate": 7.976610719679865e-05, + "loss": 8.843769137456547e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 715, + "tokens/total": 10795172, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 231159 + }, + { + "epoch": 11.1875, + "grad_norm": 0.0005816842312924564, + "learning_rate": 7.970541961511382e-05, + "loss": 1.673822225711774e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 716, + "tokens/total": 10810260, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 231480 + }, + { + "epoch": 11.203125, + "grad_norm": 0.0003033979737665504, + "learning_rate": 7.964466764523945e-05, + "loss": 1.0691360330383759e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 717, + "tokens/total": 10825452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 231806 + }, + { + "epoch": 11.21875, + "grad_norm": 0.00017789554840419441, + "learning_rate": 7.958385144550958e-05, + "loss": 6.68784741719719e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 718, + "tokens/total": 10840596, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 232128 + }, + { + "epoch": 11.234375, + "grad_norm": 0.00024442930589430034, + "learning_rate": 7.95229711744257e-05, + "loss": 8.834139407554176e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 719, + "tokens/total": 10855700, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 232446 + }, + { + "epoch": 11.25, + "grad_norm": 0.0009943818440660834, + "learning_rate": 7.946202699065622e-05, + "loss": 1.56411206262419e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 720, + "tokens/total": 10870892, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 232771 + }, + { + "epoch": 11.265625, + "grad_norm": 0.00025015894789248705, + "learning_rate": 7.940101905303615e-05, + "loss": 8.422994142165408e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 721, + "tokens/total": 10885960, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 233096 + }, + { + "epoch": 11.28125, + "grad_norm": 0.000299341743811965, + "learning_rate": 7.93399475205667e-05, + "loss": 7.969428224896546e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 722, + "tokens/total": 10900976, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 233421 + }, + { + "epoch": 11.296875, + "grad_norm": 0.00023623707238584757, + "learning_rate": 7.927881255241474e-05, + "loss": 8.238984264608007e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 723, + "tokens/total": 10916092, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 233746 + }, + { + "epoch": 11.3125, + "grad_norm": 0.0009587532840669155, + "learning_rate": 7.921761430791252e-05, + "loss": 6.994597242737655e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 724, + "tokens/total": 10931228, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 234068 + }, + { + "epoch": 11.328125, + "grad_norm": 0.0006762287230230868, + "learning_rate": 7.915635294655723e-05, + "loss": 8.578419510740787e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 725, + "tokens/total": 10946452, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 234389 + }, + { + "epoch": 11.34375, + "grad_norm": 0.0001469070848543197, + "learning_rate": 7.909502862801049e-05, + "loss": 4.493551841733279e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 726, + "tokens/total": 10961580, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 234715 + }, + { + "epoch": 11.359375, + "grad_norm": 0.000356843724148348, + "learning_rate": 7.903364151209804e-05, + "loss": 8.137749318848364e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 727, + "tokens/total": 10976596, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 235034 + }, + { + "epoch": 11.375, + "grad_norm": 0.00029630697099491954, + "learning_rate": 7.897219175880931e-05, + "loss": 8.473220077576116e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 728, + "tokens/total": 10991728, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 235355 + }, + { + "epoch": 11.390625, + "grad_norm": 0.004472525790333748, + "learning_rate": 7.89106795282969e-05, + "loss": 1.0785695849335752e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 729, + "tokens/total": 11006712, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 235674 + }, + { + "epoch": 11.40625, + "grad_norm": 0.00018268785788677633, + "learning_rate": 7.884910498087632e-05, + "loss": 5.513010819413466e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 730, + "tokens/total": 11021724, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 235999 + }, + { + "epoch": 11.421875, + "grad_norm": 0.0002784350363072008, + "learning_rate": 7.878746827702545e-05, + "loss": 6.694717285427032e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00001, + "step": 731, + "tokens/total": 11036780, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 236322 + }, + { + "epoch": 11.4375, + "grad_norm": 0.0005618222639895976, + "learning_rate": 7.872576957738418e-05, + "loss": 1.0616214240144473e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00001, + "step": 732, + "tokens/total": 11051868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 236642 + }, + { + "epoch": 11.453125, + "grad_norm": 0.00024973825202323496, + "learning_rate": 7.866400904275396e-05, + "loss": 9.546066394250374e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 733, + "tokens/total": 11066984, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 236964 + }, + { + "epoch": 11.46875, + "grad_norm": 0.00027185113867744803, + "learning_rate": 7.860218683409742e-05, + "loss": 9.028317435877398e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 734, + "tokens/total": 11082000, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 237291 + }, + { + "epoch": 11.484375, + "grad_norm": 0.00012226897524669766, + "learning_rate": 7.854030311253792e-05, + "loss": 4.406361313158413e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 735, + "tokens/total": 11097164, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 237614 + }, + { + "epoch": 11.5, + "grad_norm": 0.0001397662708768621, + "learning_rate": 7.847835803935911e-05, + "loss": 4.8795532165968325e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 736, + "tokens/total": 11112288, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 237939 + }, + { + "epoch": 11.515625, + "grad_norm": 0.0012317969230934978, + "learning_rate": 7.841635177600457e-05, + "loss": 1.585951031302102e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 737, + "tokens/total": 11127476, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 238268 + }, + { + "epoch": 11.53125, + "grad_norm": 0.00045033343485556543, + "learning_rate": 7.835428448407737e-05, + "loss": 6.74296825309284e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 738, + "tokens/total": 11142488, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 238592 + }, + { + "epoch": 11.546875, + "grad_norm": 0.00011499993706820533, + "learning_rate": 7.829215632533957e-05, + "loss": 3.6553283280227333e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 739, + "tokens/total": 11157532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 238916 + }, + { + "epoch": 11.5625, + "grad_norm": 0.00022261161939240992, + "learning_rate": 7.822996746171192e-05, + "loss": 8.351025826414116e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 740, + "tokens/total": 11172620, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 239235 + }, + { + "epoch": 11.578125, + "grad_norm": 0.0009697281057015061, + "learning_rate": 7.816771805527336e-05, + "loss": 1.1542260835994966e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 741, + "tokens/total": 11187780, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 239557 + }, + { + "epoch": 11.59375, + "grad_norm": 0.00017928672605194151, + "learning_rate": 7.810540826826064e-05, + "loss": 6.871232017147122e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 742, + "tokens/total": 11202908, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 239884 + }, + { + "epoch": 11.609375, + "grad_norm": 0.00015312970208469778, + "learning_rate": 7.804303826306784e-05, + "loss": 4.4953990254725795e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 743, + "tokens/total": 11218028, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 240200 + }, + { + "epoch": 11.625, + "grad_norm": 0.00021180683688726276, + "learning_rate": 7.798060820224602e-05, + "loss": 7.936658221296966e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 744, + "tokens/total": 11233028, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 240521 + }, + { + "epoch": 11.640625, + "grad_norm": 0.00026271751266904175, + "learning_rate": 7.791811824850271e-05, + "loss": 6.429564564314205e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 745, + "tokens/total": 11248200, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 240844 + }, + { + "epoch": 11.65625, + "grad_norm": 0.000112937756057363, + "learning_rate": 7.78555685647016e-05, + "loss": 4.1820148908300325e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 746, + "tokens/total": 11263204, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 241171 + }, + { + "epoch": 11.671875, + "grad_norm": 0.000401324505219236, + "learning_rate": 7.779295931386202e-05, + "loss": 5.586419320025016e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 747, + "tokens/total": 11278288, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 241493 + }, + { + "epoch": 11.6875, + "grad_norm": 0.0013540866784751415, + "learning_rate": 7.773029065915851e-05, + "loss": 1.639010224607773e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00002, + "step": 748, + "tokens/total": 11293364, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 241821 + }, + { + "epoch": 11.703125, + "grad_norm": 0.0008356674807146192, + "learning_rate": 7.766756276392049e-05, + "loss": 4.819988134840969e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 749, + "tokens/total": 11308524, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 242145 + }, + { + "epoch": 11.71875, + "grad_norm": 0.00012949411757290363, + "learning_rate": 7.760477579163175e-05, + "loss": 4.385303327580914e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 750, + "tokens/total": 11323636, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 242475 + }, + { + "epoch": 11.734375, + "grad_norm": 0.00033964822068810463, + "learning_rate": 7.754192990593006e-05, + "loss": 7.976723281899467e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00001, + "step": 751, + "tokens/total": 11338712, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 242796 + }, + { + "epoch": 11.75, + "grad_norm": 0.0003038443101104349, + "learning_rate": 7.747902527060669e-05, + "loss": 7.84312123869313e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 752, + "tokens/total": 11353776, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 243120 + }, + { + "epoch": 11.765625, + "grad_norm": 0.00036285549867898226, + "learning_rate": 7.741606204960606e-05, + "loss": 1.0751992704172153e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 753, + "tokens/total": 11368904, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 243445 + }, + { + "epoch": 11.78125, + "grad_norm": 0.00016507648979313672, + "learning_rate": 7.735304040702528e-05, + "loss": 5.627839527733158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 754, + "tokens/total": 11384068, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 243769 + }, + { + "epoch": 11.796875, + "grad_norm": 0.00013171533646527678, + "learning_rate": 7.728996050711372e-05, + "loss": 4.834202172787627e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 755, + "tokens/total": 11399116, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 244090 + }, + { + "epoch": 11.8125, + "grad_norm": 0.00026781720225699246, + "learning_rate": 7.722682251427256e-05, + "loss": 6.27858889856725e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 756, + "tokens/total": 11414248, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 244411 + }, + { + "epoch": 11.828125, + "grad_norm": 0.00013672966451849788, + "learning_rate": 7.716362659305444e-05, + "loss": 3.8341331674018875e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 757, + "tokens/total": 11429368, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 244739 + }, + { + "epoch": 11.84375, + "grad_norm": 0.00018275645561516285, + "learning_rate": 7.710037290816289e-05, + "loss": 5.9024632719228975e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 758, + "tokens/total": 11444340, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 245060 + }, + { + "epoch": 11.859375, + "grad_norm": 0.00012778387463185936, + "learning_rate": 7.703706162445204e-05, + "loss": 4.635352070181398e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 759, + "tokens/total": 11459428, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 245383 + }, + { + "epoch": 11.875, + "grad_norm": 0.00010651875345502049, + "learning_rate": 7.697369290692617e-05, + "loss": 3.822206508630188e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 760, + "tokens/total": 11474592, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 245704 + }, + { + "epoch": 11.890625, + "grad_norm": 0.00013545139518100768, + "learning_rate": 7.691026692073915e-05, + "loss": 4.4806192818214186e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 761, + "tokens/total": 11489644, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 246026 + }, + { + "epoch": 11.90625, + "grad_norm": 0.001958167180418968, + "learning_rate": 7.68467838311942e-05, + "loss": 2.2847019863547757e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 762, + "tokens/total": 11504716, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 246349 + }, + { + "epoch": 11.921875, + "grad_norm": 0.00014885167183820158, + "learning_rate": 7.678324380374328e-05, + "loss": 3.5454936551104765e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 763, + "tokens/total": 11519828, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 246668 + }, + { + "epoch": 11.9375, + "grad_norm": 0.00020161131396889687, + "learning_rate": 7.671964700398684e-05, + "loss": 5.791897820017766e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 764, + "tokens/total": 11534864, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 246991 + }, + { + "epoch": 11.953125, + "grad_norm": 0.00011139918206026778, + "learning_rate": 7.66559935976732e-05, + "loss": 3.7280333344824612e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 765, + "tokens/total": 11549908, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 247312 + }, + { + "epoch": 11.96875, + "grad_norm": 0.00012893143866676837, + "learning_rate": 7.659228375069827e-05, + "loss": 4.458348030311754e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 766, + "tokens/total": 11565044, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 247636 + }, + { + "epoch": 11.984375, + "grad_norm": 0.00016953289741650224, + "learning_rate": 7.652851762910505e-05, + "loss": 3.7684642393287504e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 767, + "tokens/total": 11580188, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 247959 + }, + { + "epoch": 12.0, + "grad_norm": 0.00011622429155977443, + "learning_rate": 7.646469539908316e-05, + "loss": 3.689345021484769e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 768, + "tokens/total": 11595328, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 248280 + }, + { + "epoch": 12.015625, + "grad_norm": 0.0010329362703487277, + "learning_rate": 7.640081722696851e-05, + "loss": 1.379754212393891e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 769, + "tokens/total": 11610408, + "tokens/train_per_sec_per_gpu": 8.75, + "tokens/trainable": 248603 + }, + { + "epoch": 12.03125, + "grad_norm": 8.753351721679792e-05, + "learning_rate": 7.633688327924279e-05, + "loss": 3.1169097383099142e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 770, + "tokens/total": 11625440, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 248929 + }, + { + "epoch": 12.046875, + "grad_norm": 0.00015718373470008373, + "learning_rate": 7.627289372253304e-05, + "loss": 4.225745669828029e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 771, + "tokens/total": 11640488, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 249250 + }, + { + "epoch": 12.0625, + "grad_norm": 0.00012905069161206484, + "learning_rate": 7.620884872361122e-05, + "loss": 4.561928562907269e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 772, + "tokens/total": 11655628, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 249573 + }, + { + "epoch": 12.078125, + "grad_norm": 0.000239390748902224, + "learning_rate": 7.61447484493938e-05, + "loss": 4.851561243413016e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 773, + "tokens/total": 11670680, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 249895 + }, + { + "epoch": 12.09375, + "grad_norm": 0.00010798968287417665, + "learning_rate": 7.608059306694137e-05, + "loss": 3.7757513382530306e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 774, + "tokens/total": 11685696, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 250222 + }, + { + "epoch": 12.109375, + "grad_norm": 0.00019213146879337728, + "learning_rate": 7.601638274345803e-05, + "loss": 4.42586360804853e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 775, + "tokens/total": 11700828, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 250546 + }, + { + "epoch": 12.125, + "grad_norm": 7.640149124199525e-05, + "learning_rate": 7.595211764629117e-05, + "loss": 2.4050909814832266e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 776, + "tokens/total": 11715968, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 250869 + }, + { + "epoch": 12.140625, + "grad_norm": 0.00014666939387097955, + "learning_rate": 7.588779794293086e-05, + "loss": 4.098519639228471e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 777, + "tokens/total": 11731076, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 251195 + }, + { + "epoch": 12.15625, + "grad_norm": 0.00011734114377759397, + "learning_rate": 7.582342380100953e-05, + "loss": 4.005758910352597e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 778, + "tokens/total": 11746184, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 251521 + }, + { + "epoch": 12.171875, + "grad_norm": 9.200117347063497e-05, + "learning_rate": 7.575899538830149e-05, + "loss": 3.269858098065015e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 779, + "tokens/total": 11761240, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 251840 + }, + { + "epoch": 12.1875, + "grad_norm": 6.822905561421067e-05, + "learning_rate": 7.569451287272247e-05, + "loss": 2.2363017251336714e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 780, + "tokens/total": 11776172, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 252166 + }, + { + "epoch": 12.203125, + "grad_norm": 0.00040962512139230967, + "learning_rate": 7.562997642232921e-05, + "loss": 4.020983851660276e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 781, + "tokens/total": 11791252, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 252493 + }, + { + "epoch": 12.21875, + "grad_norm": 6.947588553884998e-05, + "learning_rate": 7.556538620531904e-05, + "loss": 2.264953536723624e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 782, + "tokens/total": 11806280, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 252815 + }, + { + "epoch": 12.234375, + "grad_norm": 0.00028148273122496903, + "learning_rate": 7.55007423900294e-05, + "loss": 7.102900781319477e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 783, + "tokens/total": 11821296, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 253138 + }, + { + "epoch": 12.25, + "grad_norm": 0.0005345477839000523, + "learning_rate": 7.54360451449374e-05, + "loss": 4.053899829159491e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 784, + "tokens/total": 11836420, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 253459 + }, + { + "epoch": 12.265625, + "grad_norm": 0.00013953889720141888, + "learning_rate": 7.537129463865945e-05, + "loss": 4.29206011176575e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 785, + "tokens/total": 11851556, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 253785 + }, + { + "epoch": 12.28125, + "grad_norm": 0.0001949247089214623, + "learning_rate": 7.530649103995074e-05, + "loss": 3.856100192933809e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 786, + "tokens/total": 11866676, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 254111 + }, + { + "epoch": 12.296875, + "grad_norm": 0.0003590922278817743, + "learning_rate": 7.524163451770481e-05, + "loss": 1.007770697469823e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00001, + "step": 787, + "tokens/total": 11881856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 254433 + }, + { + "epoch": 12.3125, + "grad_norm": 0.0001303488970734179, + "learning_rate": 7.517672524095319e-05, + "loss": 4.375161097414093e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 788, + "tokens/total": 11897048, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 254756 + }, + { + "epoch": 12.328125, + "grad_norm": 0.00010425105574540794, + "learning_rate": 7.511176337886485e-05, + "loss": 3.4424488148943055e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 789, + "tokens/total": 11912072, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 255074 + }, + { + "epoch": 12.34375, + "grad_norm": 0.00011002722021657974, + "learning_rate": 7.50467491007458e-05, + "loss": 3.731329115908011e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 790, + "tokens/total": 11927120, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 255399 + }, + { + "epoch": 12.359375, + "grad_norm": 8.662894833832979e-05, + "learning_rate": 7.498168257603875e-05, + "loss": 2.95067320621456e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 791, + "tokens/total": 11942172, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 255723 + }, + { + "epoch": 12.375, + "grad_norm": 0.00018941835151053965, + "learning_rate": 7.491656397432247e-05, + "loss": 4.5850742935726885e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 792, + "tokens/total": 11957288, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 256046 + }, + { + "epoch": 12.390625, + "grad_norm": 7.976038614287972e-05, + "learning_rate": 7.485139346531149e-05, + "loss": 2.5515669221931603e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 793, + "tokens/total": 11972424, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 256368 + }, + { + "epoch": 12.40625, + "grad_norm": 9.235714969690889e-05, + "learning_rate": 7.478617121885563e-05, + "loss": 2.5488184292044025e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 794, + "tokens/total": 11987508, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 256692 + }, + { + "epoch": 12.421875, + "grad_norm": 0.00018081159214489162, + "learning_rate": 7.472089740493959e-05, + "loss": 3.463078883214621e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 795, + "tokens/total": 12002564, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 257012 + }, + { + "epoch": 12.4375, + "grad_norm": 0.00010349749936722219, + "learning_rate": 7.46555721936824e-05, + "loss": 2.919844519055914e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 796, + "tokens/total": 12017704, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 257339 + }, + { + "epoch": 12.453125, + "grad_norm": 0.00013093859888613224, + "learning_rate": 7.459019575533707e-05, + "loss": 4.470029125513975e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 797, + "tokens/total": 12032856, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 257664 + }, + { + "epoch": 12.46875, + "grad_norm": 0.00011427248682593927, + "learning_rate": 7.452476826029011e-05, + "loss": 3.6278854622651124e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 798, + "tokens/total": 12048028, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 257988 + }, + { + "epoch": 12.484375, + "grad_norm": 0.00018132558034267277, + "learning_rate": 7.445928987906114e-05, + "loss": 3.984095201303717e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 799, + "tokens/total": 12063092, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 258315 + }, + { + "epoch": 12.5, + "grad_norm": 0.0001274140231544152, + "learning_rate": 7.439376078230234e-05, + "loss": 3.1628230772184907e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 800, + "tokens/total": 12078208, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 258638 + }, + { + "epoch": 12.515625, + "grad_norm": 0.00011142557195853442, + "learning_rate": 7.432818114079813e-05, + "loss": 2.8811889478674857e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 801, + "tokens/total": 12093348, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 258963 + }, + { + "epoch": 12.53125, + "grad_norm": 0.00019037636229768395, + "learning_rate": 7.426255112546458e-05, + "loss": 4.9189375204150565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 802, + "tokens/total": 12108412, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 259287 + }, + { + "epoch": 12.546875, + "grad_norm": 0.00019567548588383943, + "learning_rate": 7.419687090734912e-05, + "loss": 5.351162144506816e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 803, + "tokens/total": 12123480, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 259607 + }, + { + "epoch": 12.5625, + "grad_norm": 6.81504825479351e-05, + "learning_rate": 7.413114065762999e-05, + "loss": 2.2098488443589304e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 804, + "tokens/total": 12138644, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 259933 + }, + { + "epoch": 12.578125, + "grad_norm": 0.00010241116979159415, + "learning_rate": 7.406536054761584e-05, + "loss": 2.833344979080721e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 805, + "tokens/total": 12153664, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 260256 + }, + { + "epoch": 12.59375, + "grad_norm": 0.00022034579887986183, + "learning_rate": 7.399953074874523e-05, + "loss": 3.5401226341491565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 806, + "tokens/total": 12168872, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 260582 + }, + { + "epoch": 12.609375, + "grad_norm": 0.00010138696961803362, + "learning_rate": 7.393365143258626e-05, + "loss": 3.6219762478140183e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 807, + "tokens/total": 12184012, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 260909 + }, + { + "epoch": 12.625, + "grad_norm": 8.012948819668964e-05, + "learning_rate": 7.386772277083606e-05, + "loss": 2.6036345843749586e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 808, + "tokens/total": 12198968, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 261237 + }, + { + "epoch": 12.640625, + "grad_norm": 9.979843162000179e-05, + "learning_rate": 7.38017449353204e-05, + "loss": 3.229568392271176e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 809, + "tokens/total": 12214048, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 261560 + }, + { + "epoch": 12.65625, + "grad_norm": 9.405840683029965e-05, + "learning_rate": 7.373571809799319e-05, + "loss": 3.057072490264545e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 810, + "tokens/total": 12229144, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 261882 + }, + { + "epoch": 12.671875, + "grad_norm": 0.000499399786349386, + "learning_rate": 7.3669642430936e-05, + "loss": 1.0245048542856239e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00001, + "step": 811, + "tokens/total": 12244368, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 262199 + }, + { + "epoch": 12.6875, + "grad_norm": 5.618187424261123e-05, + "learning_rate": 7.360351810635774e-05, + "loss": 1.7202623894263525e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 812, + "tokens/total": 12259492, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 262525 + }, + { + "epoch": 12.703125, + "grad_norm": 0.00023343712382484227, + "learning_rate": 7.353734529659409e-05, + "loss": 4.232795163261471e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 813, + "tokens/total": 12274608, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 262850 + }, + { + "epoch": 12.71875, + "grad_norm": 8.467691077385098e-05, + "learning_rate": 7.347112417410712e-05, + "loss": 2.407495458101039e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 814, + "tokens/total": 12289780, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 263171 + }, + { + "epoch": 12.734375, + "grad_norm": 0.0001598702510818839, + "learning_rate": 7.340485491148478e-05, + "loss": 3.024760644620983e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 815, + "tokens/total": 12304880, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 263495 + }, + { + "epoch": 12.75, + "grad_norm": 0.00011422912211855873, + "learning_rate": 7.33385376814405e-05, + "loss": 3.5377588574192487e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 816, + "tokens/total": 12319984, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 263818 + }, + { + "epoch": 12.765625, + "grad_norm": 4.749396612169221e-05, + "learning_rate": 7.327217265681276e-05, + "loss": 1.3899225450586528e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 817, + "tokens/total": 12335172, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 264139 + }, + { + "epoch": 12.78125, + "grad_norm": 9.546629007672891e-05, + "learning_rate": 7.320576001056452e-05, + "loss": 3.015875790879363e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 818, + "tokens/total": 12350312, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 264466 + }, + { + "epoch": 12.796875, + "grad_norm": 5.463911293190904e-05, + "learning_rate": 7.313929991578294e-05, + "loss": 1.6523881640750915e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 819, + "tokens/total": 12365496, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 264789 + }, + { + "epoch": 12.8125, + "grad_norm": 9.755798237165436e-05, + "learning_rate": 7.307279254567882e-05, + "loss": 2.3212492124002893e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 820, + "tokens/total": 12380560, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 265112 + }, + { + "epoch": 12.828125, + "grad_norm": 0.00010264299635309726, + "learning_rate": 7.300623807358614e-05, + "loss": 3.2085883958643535e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 821, + "tokens/total": 12395512, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 265435 + }, + { + "epoch": 12.84375, + "grad_norm": 8.120817801682279e-05, + "learning_rate": 7.293963667296164e-05, + "loss": 1.942413064170978e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 822, + "tokens/total": 12410528, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 265759 + }, + { + "epoch": 12.859375, + "grad_norm": 0.0003652231825981289, + "learning_rate": 7.287298851738442e-05, + "loss": 4.6293007471831515e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 823, + "tokens/total": 12425616, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 266081 + }, + { + "epoch": 12.875, + "grad_norm": 0.0002639541926328093, + "learning_rate": 7.280629378055536e-05, + "loss": 4.974856892658863e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 824, + "tokens/total": 12440816, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 266400 + }, + { + "epoch": 12.890625, + "grad_norm": 7.256776734720916e-05, + "learning_rate": 7.273955263629683e-05, + "loss": 2.3927705115056597e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 825, + "tokens/total": 12455896, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 266720 + }, + { + "epoch": 12.90625, + "grad_norm": 7.548509165644646e-05, + "learning_rate": 7.267276525855204e-05, + "loss": 2.0779016267624684e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 826, + "tokens/total": 12470980, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 267036 + }, + { + "epoch": 12.921875, + "grad_norm": 0.00012191422865726054, + "learning_rate": 7.260593182138478e-05, + "loss": 3.6637727589550195e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 827, + "tokens/total": 12486052, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 267361 + }, + { + "epoch": 12.9375, + "grad_norm": 0.0009720345260575414, + "learning_rate": 7.253905249897888e-05, + "loss": 1.0973613825626671e-05, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 828, + "tokens/total": 12501208, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 267688 + }, + { + "epoch": 12.953125, + "grad_norm": 7.413590356009081e-05, + "learning_rate": 7.24721274656377e-05, + "loss": 2.2443480247602565e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 829, + "tokens/total": 12516164, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 268009 + }, + { + "epoch": 12.96875, + "grad_norm": 0.00018752520554699004, + "learning_rate": 7.240515689578378e-05, + "loss": 4.03012245442369e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 830, + "tokens/total": 12531236, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 268332 + }, + { + "epoch": 12.984375, + "grad_norm": 7.950794679345563e-05, + "learning_rate": 7.233814096395834e-05, + "loss": 2.00104113901034e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 831, + "tokens/total": 12546304, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 268654 + }, + { + "epoch": 13.0, + "grad_norm": 0.0001989665033761412, + "learning_rate": 7.22710798448208e-05, + "loss": 5.7647207540867385e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 832, + "tokens/total": 12561532, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 268970 + }, + { + "epoch": 13.015625, + "grad_norm": 0.00012065668124705553, + "learning_rate": 7.220397371314832e-05, + "loss": 3.097037961197202e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 833, + "tokens/total": 12576708, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 269295 + }, + { + "epoch": 13.03125, + "grad_norm": 0.0001326342608081177, + "learning_rate": 7.213682274383549e-05, + "loss": 3.3605601856834255e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 834, + "tokens/total": 12591696, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 269614 + }, + { + "epoch": 13.046875, + "grad_norm": 0.00010373663099016994, + "learning_rate": 7.206962711189361e-05, + "loss": 2.7149353627464734e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 835, + "tokens/total": 12606712, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 269937 + }, + { + "epoch": 13.0625, + "grad_norm": 9.629355918150395e-05, + "learning_rate": 7.20023869924505e-05, + "loss": 3.0645442166132852e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 836, + "tokens/total": 12621748, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 270258 + }, + { + "epoch": 13.078125, + "grad_norm": 4.5782438974129036e-05, + "learning_rate": 7.193510256074983e-05, + "loss": 1.4525633105222369e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 837, + "tokens/total": 12636812, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 270586 + }, + { + "epoch": 13.09375, + "grad_norm": 0.00029143388383090496, + "learning_rate": 7.186777399215087e-05, + "loss": 3.8549328564840835e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 838, + "tokens/total": 12651848, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 270908 + }, + { + "epoch": 13.109375, + "grad_norm": 6.666341505479068e-05, + "learning_rate": 7.18004014621278e-05, + "loss": 2.207491434091935e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 839, + "tokens/total": 12666920, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 271225 + }, + { + "epoch": 13.125, + "grad_norm": 6.579246837645769e-05, + "learning_rate": 7.173298514626945e-05, + "loss": 2.0363977455417626e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 840, + "tokens/total": 12682052, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 271551 + }, + { + "epoch": 13.140625, + "grad_norm": 0.00018273077148478478, + "learning_rate": 7.166552522027874e-05, + "loss": 2.352761612200993e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 841, + "tokens/total": 12697224, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 271879 + }, + { + "epoch": 13.15625, + "grad_norm": 5.646814315696247e-05, + "learning_rate": 7.159802185997228e-05, + "loss": 1.7437188262192649e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 842, + "tokens/total": 12712252, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 272200 + }, + { + "epoch": 13.171875, + "grad_norm": 0.00011839919170597568, + "learning_rate": 7.153047524127982e-05, + "loss": 3.6436388199945213e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 843, + "tokens/total": 12727476, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 272523 + }, + { + "epoch": 13.1875, + "grad_norm": 0.0001606152654858306, + "learning_rate": 7.146288554024389e-05, + "loss": 2.839481567207258e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 844, + "tokens/total": 12742488, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 272843 + }, + { + "epoch": 13.203125, + "grad_norm": 6.378227408276871e-05, + "learning_rate": 7.139525293301933e-05, + "loss": 2.080838839901844e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 845, + "tokens/total": 12757500, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 273165 + }, + { + "epoch": 13.21875, + "grad_norm": 0.0004620689433068037, + "learning_rate": 7.132757759587276e-05, + "loss": 2.3342386157310102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 846, + "tokens/total": 12772632, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 273489 + }, + { + "epoch": 13.234375, + "grad_norm": 0.00010379543527960777, + "learning_rate": 7.125985970518216e-05, + "loss": 2.6414465992274927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 847, + "tokens/total": 12787728, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 273814 + }, + { + "epoch": 13.25, + "grad_norm": 7.535841723438352e-05, + "learning_rate": 7.119209943743645e-05, + "loss": 2.522584281905438e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 848, + "tokens/total": 12802904, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 274141 + }, + { + "epoch": 13.265625, + "grad_norm": 7.253308285726234e-05, + "learning_rate": 7.112429696923496e-05, + "loss": 2.0930983737343922e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 849, + "tokens/total": 12818004, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 274467 + }, + { + "epoch": 13.28125, + "grad_norm": 0.00010928154370049015, + "learning_rate": 7.105645247728703e-05, + "loss": 2.7142471026309067e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 850, + "tokens/total": 12833000, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 274788 + }, + { + "epoch": 13.296875, + "grad_norm": 0.0001093312821467407, + "learning_rate": 7.098856613841153e-05, + "loss": 2.9678822102141567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 851, + "tokens/total": 12848060, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 275107 + }, + { + "epoch": 13.3125, + "grad_norm": 0.00017620428116060793, + "learning_rate": 7.092063812953636e-05, + "loss": 3.679098654174595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 852, + "tokens/total": 12863264, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 275429 + }, + { + "epoch": 13.328125, + "grad_norm": 6.490548548754305e-05, + "learning_rate": 7.085266862769804e-05, + "loss": 2.1255050342006143e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 853, + "tokens/total": 12878476, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 275754 + }, + { + "epoch": 13.34375, + "grad_norm": 0.0001361994945909828, + "learning_rate": 7.078465781004124e-05, + "loss": 3.864805421471829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 854, + "tokens/total": 12893452, + "tokens/train_per_sec_per_gpu": 9.64, + "tokens/trainable": 276074 + }, + { + "epoch": 13.359375, + "grad_norm": 0.00023334445722866803, + "learning_rate": 7.071660585381828e-05, + "loss": 3.4866545775003033e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 855, + "tokens/total": 12908556, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 276397 + }, + { + "epoch": 13.375, + "grad_norm": 6.25511966063641e-05, + "learning_rate": 7.064851293638875e-05, + "loss": 1.7967286112252623e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 856, + "tokens/total": 12923656, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 276721 + }, + { + "epoch": 13.390625, + "grad_norm": 0.00011527373135322705, + "learning_rate": 7.05803792352189e-05, + "loss": 3.137032763333991e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 857, + "tokens/total": 12938808, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 277050 + }, + { + "epoch": 13.40625, + "grad_norm": 5.9854650316992775e-05, + "learning_rate": 7.05122049278814e-05, + "loss": 1.873960968623578e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 858, + "tokens/total": 12954044, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 277379 + }, + { + "epoch": 13.421875, + "grad_norm": 0.0001469842973165214, + "learning_rate": 7.044399019205467e-05, + "loss": 4.01368106395239e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 859, + "tokens/total": 12969096, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 277701 + }, + { + "epoch": 13.4375, + "grad_norm": 0.0001554029731778428, + "learning_rate": 7.037573520552248e-05, + "loss": 3.505352879074053e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 860, + "tokens/total": 12984184, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 278019 + }, + { + "epoch": 13.453125, + "grad_norm": 0.0006542939227074385, + "learning_rate": 7.030744014617355e-05, + "loss": 7.036339866317576e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 861, + "tokens/total": 12999240, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 278340 + }, + { + "epoch": 13.46875, + "grad_norm": 7.414568972308189e-05, + "learning_rate": 7.023910519200101e-05, + "loss": 1.9342680843692506e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 862, + "tokens/total": 13014336, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 278662 + }, + { + "epoch": 13.484375, + "grad_norm": 0.0002670721150934696, + "learning_rate": 7.017073052110199e-05, + "loss": 2.1539760837185895e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 863, + "tokens/total": 13029416, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 278986 + }, + { + "epoch": 13.5, + "grad_norm": 9.907476487569511e-05, + "learning_rate": 7.01023163116771e-05, + "loss": 2.038872025877936e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 864, + "tokens/total": 13044640, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 279311 + }, + { + "epoch": 13.515625, + "grad_norm": 4.2248771933373064e-05, + "learning_rate": 7.003386274203004e-05, + "loss": 1.1971330877713626e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 865, + "tokens/total": 13059684, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 279634 + }, + { + "epoch": 13.53125, + "grad_norm": 0.00012219577911309898, + "learning_rate": 6.996536999056702e-05, + "loss": 2.518343990232097e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 866, + "tokens/total": 13074760, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 279960 + }, + { + "epoch": 13.546875, + "grad_norm": 9.231813601218164e-05, + "learning_rate": 6.989683823579647e-05, + "loss": 3.010521140822675e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 867, + "tokens/total": 13089980, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 280284 + }, + { + "epoch": 13.5625, + "grad_norm": 0.00019922599312849343, + "learning_rate": 6.982826765632837e-05, + "loss": 2.023281240326469e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 868, + "tokens/total": 13104960, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 280607 + }, + { + "epoch": 13.578125, + "grad_norm": 8.861973037710413e-05, + "learning_rate": 6.975965843087396e-05, + "loss": 2.6947641345032025e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 869, + "tokens/total": 13119920, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 280927 + }, + { + "epoch": 13.59375, + "grad_norm": 9.817941463552415e-05, + "learning_rate": 6.969101073824514e-05, + "loss": 2.44341049437935e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 870, + "tokens/total": 13135080, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 281246 + }, + { + "epoch": 13.609375, + "grad_norm": 9.399205737281591e-05, + "learning_rate": 6.962232475735414e-05, + "loss": 2.4531620965717593e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 871, + "tokens/total": 13150184, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 281569 + }, + { + "epoch": 13.625, + "grad_norm": 0.00012045784387737513, + "learning_rate": 6.95536006672129e-05, + "loss": 1.9861695363943e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 872, + "tokens/total": 13165276, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 281892 + }, + { + "epoch": 13.640625, + "grad_norm": 4.345045817899518e-05, + "learning_rate": 6.948483864693274e-05, + "loss": 1.347378884020145e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 873, + "tokens/total": 13180380, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 282209 + }, + { + "epoch": 13.65625, + "grad_norm": 5.4295047448249534e-05, + "learning_rate": 6.941603887572378e-05, + "loss": 1.7618372112337966e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 874, + "tokens/total": 13195420, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 282528 + }, + { + "epoch": 13.671875, + "grad_norm": 3.7880199670325965e-05, + "learning_rate": 6.93472015328946e-05, + "loss": 1.1165370779053774e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 875, + "tokens/total": 13210508, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 282852 + }, + { + "epoch": 13.6875, + "grad_norm": 6.62184611428529e-05, + "learning_rate": 6.927832679785162e-05, + "loss": 2.0546376617858186e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 876, + "tokens/total": 13225704, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 283177 + }, + { + "epoch": 13.703125, + "grad_norm": 0.00019075955788139254, + "learning_rate": 6.92094148500988e-05, + "loss": 3.2347554679290624e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 877, + "tokens/total": 13240808, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 283502 + }, + { + "epoch": 13.71875, + "grad_norm": 7.311074296012521e-05, + "learning_rate": 6.914046586923699e-05, + "loss": 2.3871341454650974e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 878, + "tokens/total": 13255932, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 283826 + }, + { + "epoch": 13.734375, + "grad_norm": 3.782408020924777e-05, + "learning_rate": 6.907148003496365e-05, + "loss": 1.0691402394513716e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 879, + "tokens/total": 13271004, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 284153 + }, + { + "epoch": 13.75, + "grad_norm": 7.222572457976639e-05, + "learning_rate": 6.900245752707218e-05, + "loss": 2.3125546704250155e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 880, + "tokens/total": 13286120, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 284479 + }, + { + "epoch": 13.765625, + "grad_norm": 5.55890474061016e-05, + "learning_rate": 6.893339852545169e-05, + "loss": 1.693718218120921e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 881, + "tokens/total": 13301260, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 284804 + }, + { + "epoch": 13.78125, + "grad_norm": 0.00011347887630108744, + "learning_rate": 6.886430321008632e-05, + "loss": 2.710476337597356e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 882, + "tokens/total": 13316384, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 285126 + }, + { + "epoch": 13.796875, + "grad_norm": 0.00036508546327240765, + "learning_rate": 6.879517176105485e-05, + "loss": 7.0446722020278685e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 883, + "tokens/total": 13331492, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 285449 + }, + { + "epoch": 13.8125, + "grad_norm": 0.00011320950579829514, + "learning_rate": 6.872600435853025e-05, + "loss": 3.2864747936400818e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 884, + "tokens/total": 13346580, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 285770 + }, + { + "epoch": 13.828125, + "grad_norm": 5.419834633357823e-05, + "learning_rate": 6.865680118277922e-05, + "loss": 1.5860000530665275e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 885, + "tokens/total": 13361648, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 286101 + }, + { + "epoch": 13.84375, + "grad_norm": 4.323993198340759e-05, + "learning_rate": 6.858756241416163e-05, + "loss": 1.1729822517736466e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 886, + "tokens/total": 13376720, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 286424 + }, + { + "epoch": 13.859375, + "grad_norm": 5.6182903790613636e-05, + "learning_rate": 6.85182882331302e-05, + "loss": 1.382684331474593e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 887, + "tokens/total": 13391656, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 286745 + }, + { + "epoch": 13.875, + "grad_norm": 5.661810064339079e-05, + "learning_rate": 6.844897882022984e-05, + "loss": 1.716691713227192e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 888, + "tokens/total": 13406776, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 287068 + }, + { + "epoch": 13.890625, + "grad_norm": 0.00019190310558769852, + "learning_rate": 6.837963435609737e-05, + "loss": 1.7268373539991444e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 889, + "tokens/total": 13421912, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 287394 + }, + { + "epoch": 13.90625, + "grad_norm": 5.752412835136056e-05, + "learning_rate": 6.831025502146093e-05, + "loss": 1.76233402271464e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 890, + "tokens/total": 13436940, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 287718 + }, + { + "epoch": 13.921875, + "grad_norm": 6.211781146703288e-05, + "learning_rate": 6.824084099713952e-05, + "loss": 1.4081907693253015e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 891, + "tokens/total": 13452080, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 288049 + }, + { + "epoch": 13.9375, + "grad_norm": 0.0005943211144767702, + "learning_rate": 6.817139246404257e-05, + "loss": 6.830468009866308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00001, + "step": 892, + "tokens/total": 13467180, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 288368 + }, + { + "epoch": 13.953125, + "grad_norm": 5.17886619491037e-05, + "learning_rate": 6.810190960316948e-05, + "loss": 1.4416748399526114e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 893, + "tokens/total": 13482280, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 288691 + }, + { + "epoch": 13.96875, + "grad_norm": 4.485929821385071e-05, + "learning_rate": 6.803239259560903e-05, + "loss": 1.2776880566889304e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 894, + "tokens/total": 13497352, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 289015 + }, + { + "epoch": 13.984375, + "grad_norm": 5.71084747207351e-05, + "learning_rate": 6.796284162253911e-05, + "loss": 1.5132907265069662e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 895, + "tokens/total": 13512400, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 289336 + }, + { + "epoch": 14.0, + "grad_norm": 7.185615686466917e-05, + "learning_rate": 6.789325686522602e-05, + "loss": 1.277236151508987e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 896, + "tokens/total": 13527548, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 289660 + }, + { + "epoch": 14.015625, + "grad_norm": 4.978731885785237e-05, + "learning_rate": 6.782363850502417e-05, + "loss": 1.2179939403722528e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 897, + "tokens/total": 13542724, + "tokens/train_per_sec_per_gpu": 8.95, + "tokens/trainable": 289984 + }, + { + "epoch": 14.03125, + "grad_norm": 8.392137533519417e-05, + "learning_rate": 6.775398672337556e-05, + "loss": 1.799379106159904e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 898, + "tokens/total": 13557752, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 290306 + }, + { + "epoch": 14.046875, + "grad_norm": 0.0001453920267522335, + "learning_rate": 6.768430170180924e-05, + "loss": 1.989572183447308e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 899, + "tokens/total": 13572924, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 290629 + }, + { + "epoch": 14.0625, + "grad_norm": 5.724963193642907e-05, + "learning_rate": 6.761458362194097e-05, + "loss": 1.6024107480916427e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 900, + "tokens/total": 13588024, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 290959 + }, + { + "epoch": 14.078125, + "grad_norm": 8.20040368125774e-05, + "learning_rate": 6.754483266547253e-05, + "loss": 2.103687393173459e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 901, + "tokens/total": 13603244, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 291284 + }, + { + "epoch": 14.09375, + "grad_norm": 8.981539576780051e-05, + "learning_rate": 6.747504901419158e-05, + "loss": 2.5236784040316707e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 902, + "tokens/total": 13618464, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 291607 + }, + { + "epoch": 14.109375, + "grad_norm": 4.635561344912276e-05, + "learning_rate": 6.740523284997083e-05, + "loss": 1.211012204294093e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 903, + "tokens/total": 13633456, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 291932 + }, + { + "epoch": 14.125, + "grad_norm": 0.00024733011377975345, + "learning_rate": 6.733538435476779e-05, + "loss": 1.3879935067961924e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 904, + "tokens/total": 13648544, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 292255 + }, + { + "epoch": 14.140625, + "grad_norm": 0.00010022116475738585, + "learning_rate": 6.726550371062418e-05, + "loss": 1.9328476810187567e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 905, + "tokens/total": 13663696, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 292580 + }, + { + "epoch": 14.15625, + "grad_norm": 9.24205596675165e-05, + "learning_rate": 6.719559109966563e-05, + "loss": 1.4655335007773829e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 906, + "tokens/total": 13678704, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 292902 + }, + { + "epoch": 14.171875, + "grad_norm": 3.6749897844856605e-05, + "learning_rate": 6.712564670410095e-05, + "loss": 1.0849743148355628e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 907, + "tokens/total": 13693768, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 293226 + }, + { + "epoch": 14.1875, + "grad_norm": 4.9313708586851135e-05, + "learning_rate": 6.705567070622186e-05, + "loss": 1.3934755997979664e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 908, + "tokens/total": 13708908, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 293551 + }, + { + "epoch": 14.203125, + "grad_norm": 0.00010596215724945068, + "learning_rate": 6.698566328840242e-05, + "loss": 2.0530846995825414e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 909, + "tokens/total": 13724060, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 293875 + }, + { + "epoch": 14.21875, + "grad_norm": 6.745105929439887e-05, + "learning_rate": 6.69156246330986e-05, + "loss": 2.046339204753167e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 910, + "tokens/total": 13739192, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 294200 + }, + { + "epoch": 14.234375, + "grad_norm": 3.114332866971381e-05, + "learning_rate": 6.684555492284775e-05, + "loss": 7.437391218445555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 911, + "tokens/total": 13754336, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 294526 + }, + { + "epoch": 14.25, + "grad_norm": 8.269361569546163e-05, + "learning_rate": 6.677545434026816e-05, + "loss": 2.564119085946004e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 912, + "tokens/total": 13769412, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 294850 + }, + { + "epoch": 14.265625, + "grad_norm": 0.00011888529115822166, + "learning_rate": 6.670532306805861e-05, + "loss": 2.417497853457462e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 913, + "tokens/total": 13784544, + "tokens/train_per_sec_per_gpu": 8.61, + "tokens/trainable": 295174 + }, + { + "epoch": 14.28125, + "grad_norm": 6.309825403150171e-05, + "learning_rate": 6.663516128899785e-05, + "loss": 1.8808964341587853e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 914, + "tokens/total": 13799696, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 295496 + }, + { + "epoch": 14.296875, + "grad_norm": 5.9262347349431366e-05, + "learning_rate": 6.656496918594413e-05, + "loss": 1.7357599517708877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 915, + "tokens/total": 13814812, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 295825 + }, + { + "epoch": 14.3125, + "grad_norm": 5.164822141523473e-05, + "learning_rate": 6.649474694183474e-05, + "loss": 1.2478254802772426e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 916, + "tokens/total": 13829944, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 296147 + }, + { + "epoch": 14.328125, + "grad_norm": 6.594896694878116e-05, + "learning_rate": 6.642449473968551e-05, + "loss": 1.6502298194609466e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 917, + "tokens/total": 13845084, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 296470 + }, + { + "epoch": 14.34375, + "grad_norm": 5.508278263732791e-05, + "learning_rate": 6.635421276259038e-05, + "loss": 1.3780652352579636e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 918, + "tokens/total": 13860172, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 296797 + }, + { + "epoch": 14.359375, + "grad_norm": 5.368686106521636e-05, + "learning_rate": 6.628390119372085e-05, + "loss": 1.5078080650710035e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 919, + "tokens/total": 13875296, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 297118 + }, + { + "epoch": 14.375, + "grad_norm": 4.166882354184054e-05, + "learning_rate": 6.621356021632556e-05, + "loss": 1.2374182460916927e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 920, + "tokens/total": 13890396, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 297441 + }, + { + "epoch": 14.390625, + "grad_norm": 4.058206468471326e-05, + "learning_rate": 6.614319001372981e-05, + "loss": 1.086644488168531e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 921, + "tokens/total": 13905476, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 297763 + }, + { + "epoch": 14.40625, + "grad_norm": 4.9323389248456806e-05, + "learning_rate": 6.607279076933509e-05, + "loss": 1.4575615523426677e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 922, + "tokens/total": 13920536, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 298088 + }, + { + "epoch": 14.421875, + "grad_norm": 3.589461266528815e-05, + "learning_rate": 6.600236266661848e-05, + "loss": 8.809178098090342e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 923, + "tokens/total": 13935580, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 298411 + }, + { + "epoch": 14.4375, + "grad_norm": 8.650182280689478e-05, + "learning_rate": 6.593190588913239e-05, + "loss": 1.5303246527764713e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 924, + "tokens/total": 13950700, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 298733 + }, + { + "epoch": 14.453125, + "grad_norm": 8.58692146721296e-05, + "learning_rate": 6.586142062050387e-05, + "loss": 2.0843222046096344e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 925, + "tokens/total": 13965820, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 299054 + }, + { + "epoch": 14.46875, + "grad_norm": 6.944965571165085e-05, + "learning_rate": 6.579090704443432e-05, + "loss": 1.1496265415189555e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 926, + "tokens/total": 13980948, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 299375 + }, + { + "epoch": 14.484375, + "grad_norm": 5.2342576964292675e-05, + "learning_rate": 6.572036534469879e-05, + "loss": 1.2702308822554187e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 927, + "tokens/total": 13996084, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 299700 + }, + { + "epoch": 14.5, + "grad_norm": 5.443696500151418e-05, + "learning_rate": 6.564979570514577e-05, + "loss": 1.7568534076417563e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 928, + "tokens/total": 14011092, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 300022 + }, + { + "epoch": 14.515625, + "grad_norm": 0.00025401683524250984, + "learning_rate": 6.557919830969644e-05, + "loss": 4.309782980271848e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 929, + "tokens/total": 14026324, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 300348 + }, + { + "epoch": 14.53125, + "grad_norm": 7.833947893232107e-05, + "learning_rate": 6.550857334234442e-05, + "loss": 1.8433300965625676e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 930, + "tokens/total": 14041448, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 300671 + }, + { + "epoch": 14.546875, + "grad_norm": 8.168330532498658e-05, + "learning_rate": 6.543792098715513e-05, + "loss": 1.8892920934376889e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 931, + "tokens/total": 14056636, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 300994 + }, + { + "epoch": 14.5625, + "grad_norm": 5.483560016728006e-05, + "learning_rate": 6.536724142826536e-05, + "loss": 1.629071448405739e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 932, + "tokens/total": 14071748, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 301323 + }, + { + "epoch": 14.578125, + "grad_norm": 4.461768548935652e-05, + "learning_rate": 6.529653484988287e-05, + "loss": 1.0997707704518689e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 933, + "tokens/total": 14086812, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 301649 + }, + { + "epoch": 14.59375, + "grad_norm": 0.0001430799311492592, + "learning_rate": 6.522580143628573e-05, + "loss": 2.7947155558649683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 934, + "tokens/total": 14101904, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 301975 + }, + { + "epoch": 14.609375, + "grad_norm": 8.027078001759946e-05, + "learning_rate": 6.515504137182206e-05, + "loss": 1.9248398075433215e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 935, + "tokens/total": 14117012, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 302295 + }, + { + "epoch": 14.625, + "grad_norm": 9.045998012879863e-05, + "learning_rate": 6.508425484090935e-05, + "loss": 2.628022912176675e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 936, + "tokens/total": 14131976, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 302617 + }, + { + "epoch": 14.640625, + "grad_norm": 4.7691853978903964e-05, + "learning_rate": 6.501344202803414e-05, + "loss": 1.1184176855749683e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 937, + "tokens/total": 14147008, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 302937 + }, + { + "epoch": 14.65625, + "grad_norm": 8.31821234896779e-05, + "learning_rate": 6.494260311775145e-05, + "loss": 2.248458258691244e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 938, + "tokens/total": 14162152, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 303265 + }, + { + "epoch": 14.671875, + "grad_norm": 7.890448614489287e-05, + "learning_rate": 6.487173829468424e-05, + "loss": 1.7639106317801634e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 939, + "tokens/total": 14177152, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 303589 + }, + { + "epoch": 14.6875, + "grad_norm": 0.0007585128187201917, + "learning_rate": 6.480084774352312e-05, + "loss": 9.449095159652643e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 940, + "tokens/total": 14192124, + "tokens/train_per_sec_per_gpu": 8.68, + "tokens/trainable": 303911 + }, + { + "epoch": 14.703125, + "grad_norm": 4.063158485223539e-05, + "learning_rate": 6.472993164902566e-05, + "loss": 1.0883406957873376e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 941, + "tokens/total": 14207176, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 304235 + }, + { + "epoch": 14.71875, + "grad_norm": 0.0012726637069135904, + "learning_rate": 6.465899019601607e-05, + "loss": 1.647085127842729e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 942, + "tokens/total": 14222308, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 304559 + }, + { + "epoch": 14.734375, + "grad_norm": 6.790161569369957e-05, + "learning_rate": 6.458802356938462e-05, + "loss": 2.0932579900545534e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 943, + "tokens/total": 14237520, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 304880 + }, + { + "epoch": 14.75, + "grad_norm": 5.292199057294056e-05, + "learning_rate": 6.451703195408714e-05, + "loss": 1.5438358786923345e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 944, + "tokens/total": 14252592, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 305200 + }, + { + "epoch": 14.765625, + "grad_norm": 6.903067696839571e-05, + "learning_rate": 6.444601553514472e-05, + "loss": 1.4309335938378354e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 945, + "tokens/total": 14267684, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 305522 + }, + { + "epoch": 14.78125, + "grad_norm": 6.407395994756371e-05, + "learning_rate": 6.437497449764292e-05, + "loss": 1.7573401009940426e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 946, + "tokens/total": 14282812, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 305844 + }, + { + "epoch": 14.796875, + "grad_norm": 7.578920485684648e-05, + "learning_rate": 6.43039090267316e-05, + "loss": 1.912631887535099e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 947, + "tokens/total": 14298036, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 306167 + }, + { + "epoch": 14.8125, + "grad_norm": 6.992624548729509e-05, + "learning_rate": 6.423281930762426e-05, + "loss": 1.189959675684804e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 948, + "tokens/total": 14313140, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 306489 + }, + { + "epoch": 14.828125, + "grad_norm": 6.511853280244395e-05, + "learning_rate": 6.416170552559758e-05, + "loss": 1.474600708206708e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 949, + "tokens/total": 14328408, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 306810 + }, + { + "epoch": 14.84375, + "grad_norm": 0.00010607110743876547, + "learning_rate": 6.409056786599093e-05, + "loss": 2.453754404996289e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 950, + "tokens/total": 14343448, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 307130 + }, + { + "epoch": 14.859375, + "grad_norm": 0.0001000507763819769, + "learning_rate": 6.401940651420598e-05, + "loss": 2.529926860006526e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 951, + "tokens/total": 14358560, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 307453 + }, + { + "epoch": 14.875, + "grad_norm": 0.00011187302879989147, + "learning_rate": 6.39482216557061e-05, + "loss": 2.2469866962637752e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 952, + "tokens/total": 14373584, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 307776 + }, + { + "epoch": 14.890625, + "grad_norm": 4.595956488628872e-05, + "learning_rate": 6.387701347601593e-05, + "loss": 1.499669224358513e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 953, + "tokens/total": 14388648, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 308096 + }, + { + "epoch": 14.90625, + "grad_norm": 6.818763358751312e-05, + "learning_rate": 6.380578216072088e-05, + "loss": 8.027869853322045e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 954, + "tokens/total": 14403620, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 308416 + }, + { + "epoch": 14.921875, + "grad_norm": 0.0001449510018574074, + "learning_rate": 6.373452789546669e-05, + "loss": 2.331412360945251e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 955, + "tokens/total": 14418724, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 308740 + }, + { + "epoch": 14.9375, + "grad_norm": 3.8131242035888135e-05, + "learning_rate": 6.36632508659589e-05, + "loss": 9.072298894352571e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 956, + "tokens/total": 14433808, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 309062 + }, + { + "epoch": 14.953125, + "grad_norm": 2.594317265902646e-05, + "learning_rate": 6.359195125796233e-05, + "loss": 6.713360676258162e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 957, + "tokens/total": 14448956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 309391 + }, + { + "epoch": 14.96875, + "grad_norm": 0.00010180116805713624, + "learning_rate": 6.352062925730074e-05, + "loss": 2.0256127299944637e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 958, + "tokens/total": 14463960, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 309716 + }, + { + "epoch": 14.984375, + "grad_norm": 3.978185850428417e-05, + "learning_rate": 6.344928504985616e-05, + "loss": 1.1063133342759102e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 959, + "tokens/total": 14479012, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 310029 + }, + { + "epoch": 15.0, + "grad_norm": 4.233407889842056e-05, + "learning_rate": 6.337791882156857e-05, + "loss": 1.1700456070684595e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 960, + "tokens/total": 14494116, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 310350 + }, + { + "epoch": 15.015625, + "grad_norm": 6.0397691413527355e-05, + "learning_rate": 6.330653075843527e-05, + "loss": 1.8204215166406357e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 961, + "tokens/total": 14509288, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 310676 + }, + { + "epoch": 15.03125, + "grad_norm": 4.4577634980669245e-05, + "learning_rate": 6.323512104651052e-05, + "loss": 1.2006689757981803e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 962, + "tokens/total": 14524308, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 310999 + }, + { + "epoch": 15.046875, + "grad_norm": 4.363745028967969e-05, + "learning_rate": 6.316368987190498e-05, + "loss": 1.057101371770841e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 963, + "tokens/total": 14539320, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 311326 + }, + { + "epoch": 15.0625, + "grad_norm": 4.965631160303019e-05, + "learning_rate": 6.309223742078527e-05, + "loss": 1.4942244206395117e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 964, + "tokens/total": 14554348, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 311652 + }, + { + "epoch": 15.078125, + "grad_norm": 4.124893530388363e-05, + "learning_rate": 6.302076387937343e-05, + "loss": 1.0145915894099744e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 965, + "tokens/total": 14569396, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 311979 + }, + { + "epoch": 15.09375, + "grad_norm": 0.00016004100325517356, + "learning_rate": 6.294926943394646e-05, + "loss": 2.7917315037484514e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 966, + "tokens/total": 14584500, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 312300 + }, + { + "epoch": 15.109375, + "grad_norm": 5.61064480280038e-05, + "learning_rate": 6.28777542708359e-05, + "loss": 1.3850698223905056e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 967, + "tokens/total": 14599592, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 312620 + }, + { + "epoch": 15.125, + "grad_norm": 6.103901250753552e-05, + "learning_rate": 6.280621857642722e-05, + "loss": 1.4942685311325477e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 968, + "tokens/total": 14614736, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 312944 + }, + { + "epoch": 15.140625, + "grad_norm": 2.391202906437684e-05, + "learning_rate": 6.273466253715941e-05, + "loss": 6.849907094874652e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 969, + "tokens/total": 14629748, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 313271 + }, + { + "epoch": 15.15625, + "grad_norm": 4.6358443796634674e-05, + "learning_rate": 6.266308633952454e-05, + "loss": 1.3744077023147838e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 970, + "tokens/total": 14644736, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 313592 + }, + { + "epoch": 15.171875, + "grad_norm": 3.74452029063832e-05, + "learning_rate": 6.259149017006713e-05, + "loss": 8.902447348191345e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 971, + "tokens/total": 14659816, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 313918 + }, + { + "epoch": 15.1875, + "grad_norm": 9.493020479567349e-05, + "learning_rate": 6.251987421538383e-05, + "loss": 1.6062704162322916e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 972, + "tokens/total": 14674920, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 314244 + }, + { + "epoch": 15.203125, + "grad_norm": 2.5273504434153438e-05, + "learning_rate": 6.244823866212282e-05, + "loss": 7.111624995559396e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 973, + "tokens/total": 14690064, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 314568 + }, + { + "epoch": 15.21875, + "grad_norm": 0.00010362095781601965, + "learning_rate": 6.237658369698335e-05, + "loss": 1.7851814391178777e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 974, + "tokens/total": 14705236, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 314888 + }, + { + "epoch": 15.234375, + "grad_norm": 6.892029341543093e-05, + "learning_rate": 6.23049095067153e-05, + "loss": 8.738239785088808e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 975, + "tokens/total": 14720384, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 315217 + }, + { + "epoch": 15.25, + "grad_norm": 0.00016985680849757046, + "learning_rate": 6.223321627811857e-05, + "loss": 1.4400820873561315e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 976, + "tokens/total": 14735432, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 315538 + }, + { + "epoch": 15.265625, + "grad_norm": 4.9234015023102984e-05, + "learning_rate": 6.216150419804279e-05, + "loss": 1.3229498563305242e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 977, + "tokens/total": 14750520, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 315862 + }, + { + "epoch": 15.28125, + "grad_norm": 6.438756099669263e-05, + "learning_rate": 6.208977345338665e-05, + "loss": 1.5429411632794654e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 978, + "tokens/total": 14765648, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 316185 + }, + { + "epoch": 15.296875, + "grad_norm": 3.929212471120991e-05, + "learning_rate": 6.201802423109749e-05, + "loss": 1.100590338865004e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 979, + "tokens/total": 14780724, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 316510 + }, + { + "epoch": 15.3125, + "grad_norm": 6.362700514728203e-05, + "learning_rate": 6.194625671817084e-05, + "loss": 1.5663575823054998e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 980, + "tokens/total": 14795824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 316830 + }, + { + "epoch": 15.328125, + "grad_norm": 5.856006464455277e-05, + "learning_rate": 6.187447110164986e-05, + "loss": 9.909776963468175e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 981, + "tokens/total": 14810876, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 317153 + }, + { + "epoch": 15.34375, + "grad_norm": 4.1728420910658315e-05, + "learning_rate": 6.180266756862493e-05, + "loss": 9.539714938000543e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 982, + "tokens/total": 14825992, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 317473 + }, + { + "epoch": 15.359375, + "grad_norm": 3.521126927807927e-05, + "learning_rate": 6.173084630623311e-05, + "loss": 9.559518048263271e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 983, + "tokens/total": 14840956, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 317796 + }, + { + "epoch": 15.375, + "grad_norm": 2.665440842974931e-05, + "learning_rate": 6.165900750165763e-05, + "loss": 6.843700930403429e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 984, + "tokens/total": 14856184, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 318123 + }, + { + "epoch": 15.390625, + "grad_norm": 7.02376346453093e-05, + "learning_rate": 6.15871513421275e-05, + "loss": 1.4553772871295223e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 985, + "tokens/total": 14871248, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 318449 + }, + { + "epoch": 15.40625, + "grad_norm": 8.755866292631254e-05, + "learning_rate": 6.151527801491692e-05, + "loss": 1.3671403849002672e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 986, + "tokens/total": 14886400, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 318775 + }, + { + "epoch": 15.421875, + "grad_norm": 6.377039971994236e-05, + "learning_rate": 6.144338770734487e-05, + "loss": 1.655393248256587e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 987, + "tokens/total": 14901536, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 319093 + }, + { + "epoch": 15.4375, + "grad_norm": 5.989666533423588e-05, + "learning_rate": 6.137148060677453e-05, + "loss": 9.670616236689966e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 988, + "tokens/total": 14916616, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 319418 + }, + { + "epoch": 15.453125, + "grad_norm": 5.5695742048555985e-05, + "learning_rate": 6.129955690061289e-05, + "loss": 8.72893622272386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 989, + "tokens/total": 14931756, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 319739 + }, + { + "epoch": 15.46875, + "grad_norm": 3.395605745026842e-05, + "learning_rate": 6.122761677631022e-05, + "loss": 7.767515057821583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 990, + "tokens/total": 14946856, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 320062 + }, + { + "epoch": 15.484375, + "grad_norm": 0.00013859746104571968, + "learning_rate": 6.115566042135953e-05, + "loss": 2.581086846475955e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 991, + "tokens/total": 14962100, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 320385 + }, + { + "epoch": 15.5, + "grad_norm": 3.372377614141442e-05, + "learning_rate": 6.10836880232962e-05, + "loss": 8.867686460689583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 992, + "tokens/total": 14977184, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 320706 + }, + { + "epoch": 15.515625, + "grad_norm": 6.984758510952815e-05, + "learning_rate": 6.1011699769697384e-05, + "loss": 1.2891443930129753e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 993, + "tokens/total": 14992300, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 321031 + }, + { + "epoch": 15.53125, + "grad_norm": 5.5518965382361785e-05, + "learning_rate": 6.0939695848181546e-05, + "loss": 1.398159497512097e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 994, + "tokens/total": 15007460, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 321349 + }, + { + "epoch": 15.546875, + "grad_norm": 8.458229422103614e-05, + "learning_rate": 6.086767644640801e-05, + "loss": 1.4708990647704923e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 995, + "tokens/total": 15022532, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 321672 + }, + { + "epoch": 15.5625, + "grad_norm": 0.00017973266949411482, + "learning_rate": 6.079564175207645e-05, + "loss": 3.492345513222972e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 996, + "tokens/total": 15037596, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 321995 + }, + { + "epoch": 15.578125, + "grad_norm": 5.182886525290087e-05, + "learning_rate": 6.072359195292635e-05, + "loss": 1.2431519280653447e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 997, + "tokens/total": 15052708, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 322320 + }, + { + "epoch": 15.59375, + "grad_norm": 0.00039384455885738134, + "learning_rate": 6.0651527236736636e-05, + "loss": 4.331072432250949e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 998, + "tokens/total": 15067820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 322642 + }, + { + "epoch": 15.609375, + "grad_norm": 6.89527005306445e-05, + "learning_rate": 6.057944779132505e-05, + "loss": 1.9015310499526095e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 999, + "tokens/total": 15082956, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 322968 + }, + { + "epoch": 15.625, + "grad_norm": 0.00038482208037748933, + "learning_rate": 6.050735380454774e-05, + "loss": 4.186047590337694e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1000, + "tokens/total": 15098020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 323293 + }, + { + "epoch": 15.640625, + "grad_norm": 2.5691933842608705e-05, + "learning_rate": 6.043524546429874e-05, + "loss": 7.110375008778647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1001, + "tokens/total": 15113016, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 323615 + }, + { + "epoch": 15.65625, + "grad_norm": 5.0753205869114026e-05, + "learning_rate": 6.036312295850954e-05, + "loss": 1.0369699339207727e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1002, + "tokens/total": 15128136, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 323945 + }, + { + "epoch": 15.671875, + "grad_norm": 1.725030415400397e-05, + "learning_rate": 6.029098647514848e-05, + "loss": 4.307669883019116e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1003, + "tokens/total": 15143328, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 324273 + }, + { + "epoch": 15.6875, + "grad_norm": 5.06673677591607e-05, + "learning_rate": 6.0218836202220394e-05, + "loss": 1.4467525488726096e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1004, + "tokens/total": 15158384, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 324598 + }, + { + "epoch": 15.703125, + "grad_norm": 2.792146915453486e-05, + "learning_rate": 6.014667232776601e-05, + "loss": 7.40162306556158e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1005, + "tokens/total": 15173492, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 324921 + }, + { + "epoch": 15.71875, + "grad_norm": 3.8122736441437155e-05, + "learning_rate": 6.0074495039861515e-05, + "loss": 1.123777792599867e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1006, + "tokens/total": 15188732, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 325244 + }, + { + "epoch": 15.734375, + "grad_norm": 8.14247687230818e-05, + "learning_rate": 6.000230452661806e-05, + "loss": 1.9589356270444114e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1007, + "tokens/total": 15203780, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 325567 + }, + { + "epoch": 15.75, + "grad_norm": 5.886691724299453e-05, + "learning_rate": 5.993010097618127e-05, + "loss": 1.378559204567864e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1008, + "tokens/total": 15218936, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 325892 + }, + { + "epoch": 15.765625, + "grad_norm": 0.00011032194015569985, + "learning_rate": 5.985788457673074e-05, + "loss": 1.073725343303522e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1009, + "tokens/total": 15234040, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 326212 + }, + { + "epoch": 15.78125, + "grad_norm": 0.00013100061914883554, + "learning_rate": 5.9785655516479535e-05, + "loss": 2.953038574560196e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1010, + "tokens/total": 15249068, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 326535 + }, + { + "epoch": 15.796875, + "grad_norm": 4.720654396805912e-05, + "learning_rate": 5.9713413983673774e-05, + "loss": 7.844298011150386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1011, + "tokens/total": 15264252, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 326858 + }, + { + "epoch": 15.8125, + "grad_norm": 3.831119465758093e-05, + "learning_rate": 5.9641160166592026e-05, + "loss": 1.0103673275807523e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1012, + "tokens/total": 15279436, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 327178 + }, + { + "epoch": 15.828125, + "grad_norm": 4.244214505888522e-05, + "learning_rate": 5.9568894253544874e-05, + "loss": 1.1848846952489112e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1013, + "tokens/total": 15294388, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 327502 + }, + { + "epoch": 15.84375, + "grad_norm": 4.1179850086336955e-05, + "learning_rate": 5.9496616432874466e-05, + "loss": 1.0905631597779575e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1014, + "tokens/total": 15309404, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 327821 + }, + { + "epoch": 15.859375, + "grad_norm": 4.633567368728109e-05, + "learning_rate": 5.9424326892953975e-05, + "loss": 1.24127450362721e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1015, + "tokens/total": 15324568, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 328143 + }, + { + "epoch": 15.875, + "grad_norm": 3.0212768251658417e-05, + "learning_rate": 5.935202582218707e-05, + "loss": 8.720062965039688e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1016, + "tokens/total": 15339560, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 328462 + }, + { + "epoch": 15.890625, + "grad_norm": 7.697457476751879e-05, + "learning_rate": 5.9279713409007555e-05, + "loss": 1.3978632296129945e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1017, + "tokens/total": 15354696, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 328785 + }, + { + "epoch": 15.90625, + "grad_norm": 3.8135396607685834e-05, + "learning_rate": 5.92073898418787e-05, + "loss": 1.062007299879042e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1018, + "tokens/total": 15369816, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 329108 + }, + { + "epoch": 15.921875, + "grad_norm": 3.6240795452613384e-05, + "learning_rate": 5.9135055309292955e-05, + "loss": 9.362717605654325e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1019, + "tokens/total": 15384936, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 329429 + }, + { + "epoch": 15.9375, + "grad_norm": 3.131631819996983e-05, + "learning_rate": 5.906270999977121e-05, + "loss": 8.221573466471455e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1020, + "tokens/total": 15400068, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 329751 + }, + { + "epoch": 15.953125, + "grad_norm": 0.00010301479778718203, + "learning_rate": 5.899035410186259e-05, + "loss": 1.3009018857701449e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1021, + "tokens/total": 15415120, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 330075 + }, + { + "epoch": 15.96875, + "grad_norm": 3.9301598008023575e-05, + "learning_rate": 5.891798780414369e-05, + "loss": 1.0615477776809712e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1022, + "tokens/total": 15430256, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 330394 + }, + { + "epoch": 15.984375, + "grad_norm": 9.136048902291805e-05, + "learning_rate": 5.884561129521832e-05, + "loss": 8.340676345142128e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1023, + "tokens/total": 15445400, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 330717 + }, + { + "epoch": 16.0, + "grad_norm": 4.0084210922941566e-05, + "learning_rate": 5.8773224763716795e-05, + "loss": 7.630780487488664e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1024, + "tokens/total": 15460436, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 331040 + }, + { + "epoch": 16.015625, + "grad_norm": 5.882185359951109e-05, + "learning_rate": 5.8700828398295636e-05, + "loss": 1.0005110198108014e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1025, + "tokens/total": 15475588, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 331364 + }, + { + "epoch": 16.03125, + "grad_norm": 3.843717786367051e-05, + "learning_rate": 5.862842238763694e-05, + "loss": 9.751887546372018e-07, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1026, + "tokens/total": 15490656, + "tokens/train_per_sec_per_gpu": 8.89, + "tokens/trainable": 331683 + }, + { + "epoch": 16.046875, + "grad_norm": 2.279582986375317e-05, + "learning_rate": 5.855600692044797e-05, + "loss": 6.278231694523129e-07, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1027, + "tokens/total": 15505588, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 332008 + }, + { + "epoch": 16.0625, + "grad_norm": 5.805299588246271e-05, + "learning_rate": 5.8483582185460636e-05, + "loss": 1.191105639009038e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1028, + "tokens/total": 15520748, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 332331 + }, + { + "epoch": 16.078125, + "grad_norm": 0.00016670458717271686, + "learning_rate": 5.8411148371431e-05, + "loss": 2.9001751045143465e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1029, + "tokens/total": 15535884, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 332654 + }, + { + "epoch": 16.09375, + "grad_norm": 3.082956754951738e-05, + "learning_rate": 5.833870566713876e-05, + "loss": 8.530229251846322e-07, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1030, + "tokens/total": 15551104, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 332976 + }, + { + "epoch": 16.109375, + "grad_norm": 5.893274283153005e-05, + "learning_rate": 5.826625426138681e-05, + "loss": 9.321727816313796e-07, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1031, + "tokens/total": 15566260, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 333299 + }, + { + "epoch": 16.125, + "grad_norm": 4.112880924367346e-05, + "learning_rate": 5.8193794343000715e-05, + "loss": 1.0188530268351315e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1032, + "tokens/total": 15581488, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 333624 + }, + { + "epoch": 16.140625, + "grad_norm": 2.3491005777032115e-05, + "learning_rate": 5.812132610082825e-05, + "loss": 5.011782491237682e-07, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1033, + "tokens/total": 15596436, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 333946 + }, + { + "epoch": 16.15625, + "grad_norm": 8.688800153322518e-05, + "learning_rate": 5.804884972373882e-05, + "loss": 1.5065056686580647e-06, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1034, + "tokens/total": 15611532, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 334268 + }, + { + "epoch": 16.171875, + "grad_norm": 3.554632712621242e-05, + "learning_rate": 5.79763654006231e-05, + "loss": 9.921500350174028e-07, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1035, + "tokens/total": 15626552, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 334588 + }, + { + "epoch": 16.1875, + "grad_norm": 2.8848417059634812e-05, + "learning_rate": 5.790387332039244e-05, + "loss": 6.414871904780739e-07, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1036, + "tokens/total": 15641640, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 334911 + }, + { + "epoch": 16.203125, + "grad_norm": 5.013435293221846e-05, + "learning_rate": 5.783137367197842e-05, + "loss": 1.2875902939413209e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1037, + "tokens/total": 15656820, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 335233 + }, + { + "epoch": 16.21875, + "grad_norm": 0.0003741619293577969, + "learning_rate": 5.775886664433232e-05, + "loss": 3.661276878119679e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1038, + "tokens/total": 15671984, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 335555 + }, + { + "epoch": 16.234375, + "grad_norm": 2.7448808396002278e-05, + "learning_rate": 5.7686352426424696e-05, + "loss": 7.290456665032252e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1039, + "tokens/total": 15687116, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 335882 + }, + { + "epoch": 16.25, + "grad_norm": 2.7621319532045163e-05, + "learning_rate": 5.76138312072448e-05, + "loss": 7.033851829874038e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1040, + "tokens/total": 15702184, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 336205 + }, + { + "epoch": 16.265625, + "grad_norm": 3.5506953281583264e-05, + "learning_rate": 5.7541303175800156e-05, + "loss": 9.121825996771804e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1041, + "tokens/total": 15717188, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 336528 + }, + { + "epoch": 16.28125, + "grad_norm": 3.4210796002298594e-05, + "learning_rate": 5.746876852111602e-05, + "loss": 8.357579872608767e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1042, + "tokens/total": 15732344, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 336855 + }, + { + "epoch": 16.296875, + "grad_norm": 2.9033575629000552e-05, + "learning_rate": 5.739622743223494e-05, + "loss": 5.414926818048116e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1043, + "tokens/total": 15747420, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 337177 + }, + { + "epoch": 16.3125, + "grad_norm": 6.592264981009066e-05, + "learning_rate": 5.7323680098216215e-05, + "loss": 1.6729719618524541e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1044, + "tokens/total": 15762480, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 337498 + }, + { + "epoch": 16.328125, + "grad_norm": 2.8539443519548513e-05, + "learning_rate": 5.725112670813543e-05, + "loss": 7.429445645357191e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1045, + "tokens/total": 15777524, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 337819 + }, + { + "epoch": 16.34375, + "grad_norm": 3.7551388231804594e-05, + "learning_rate": 5.717856745108393e-05, + "loss": 9.27614678403188e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1046, + "tokens/total": 15792672, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 338138 + }, + { + "epoch": 16.359375, + "grad_norm": 0.00011167993216076866, + "learning_rate": 5.710600251616838e-05, + "loss": 8.151945394274662e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1047, + "tokens/total": 15807740, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 338462 + }, + { + "epoch": 16.375, + "grad_norm": 4.584225098369643e-05, + "learning_rate": 5.703343209251023e-05, + "loss": 9.37580807658378e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1048, + "tokens/total": 15822804, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 338787 + }, + { + "epoch": 16.390625, + "grad_norm": 3.384376032045111e-05, + "learning_rate": 5.696085636924522e-05, + "loss": 9.14638349058805e-07, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1049, + "tokens/total": 15837876, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 339115 + }, + { + "epoch": 16.40625, + "grad_norm": 5.283071368467063e-05, + "learning_rate": 5.688827553552294e-05, + "loss": 1.1609848797888844e-06, + "memory/device_reserved (GiB)": 34.17, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1050, + "tokens/total": 15853020, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 339438 + }, + { + "epoch": 16.421875, + "grad_norm": 7.525815453846008e-05, + "learning_rate": 5.681568978050624e-05, + "loss": 1.1107866839665803e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1051, + "tokens/total": 15868256, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 339761 + }, + { + "epoch": 16.4375, + "grad_norm": 2.44142775045475e-05, + "learning_rate": 5.6743099293370884e-05, + "loss": 6.146544819785049e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1052, + "tokens/total": 15883452, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 340082 + }, + { + "epoch": 16.453125, + "grad_norm": 9.277810750063509e-05, + "learning_rate": 5.667050426330488e-05, + "loss": 1.3340549003260094e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1053, + "tokens/total": 15898444, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 340401 + }, + { + "epoch": 16.46875, + "grad_norm": 6.511708488687873e-05, + "learning_rate": 5.659790487950812e-05, + "loss": 1.5211393247227534e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1054, + "tokens/total": 15913516, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 340725 + }, + { + "epoch": 16.484375, + "grad_norm": 5.15691535838414e-05, + "learning_rate": 5.652530133119183e-05, + "loss": 1.3950009360996773e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1055, + "tokens/total": 15928648, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 341046 + }, + { + "epoch": 16.5, + "grad_norm": 3.580413613235578e-05, + "learning_rate": 5.645269380757813e-05, + "loss": 9.06785999177373e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1056, + "tokens/total": 15943728, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 341370 + }, + { + "epoch": 16.515625, + "grad_norm": 6.294550257734954e-05, + "learning_rate": 5.6380082497899414e-05, + "loss": 1.4009697224537376e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1057, + "tokens/total": 15958892, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 341695 + }, + { + "epoch": 16.53125, + "grad_norm": 2.7993484764010645e-05, + "learning_rate": 5.630746759139803e-05, + "loss": 6.4136463606701e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1058, + "tokens/total": 15974052, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 342017 + }, + { + "epoch": 16.546875, + "grad_norm": 3.616355388658121e-05, + "learning_rate": 5.6234849277325675e-05, + "loss": 1.0017855629484984e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1059, + "tokens/total": 15989292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 342346 + }, + { + "epoch": 16.5625, + "grad_norm": 5.919004979659803e-05, + "learning_rate": 5.616222774494289e-05, + "loss": 1.6761098322604084e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1060, + "tokens/total": 16004404, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 342672 + }, + { + "epoch": 16.578125, + "grad_norm": 0.00018406064191367477, + "learning_rate": 5.608960318351863e-05, + "loss": 8.658131491756649e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1061, + "tokens/total": 16019580, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 342996 + }, + { + "epoch": 16.59375, + "grad_norm": 5.50741606275551e-05, + "learning_rate": 5.601697578232978e-05, + "loss": 7.080702175699116e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1062, + "tokens/total": 16034716, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 343322 + }, + { + "epoch": 16.609375, + "grad_norm": 2.1674990421161056e-05, + "learning_rate": 5.594434573066056e-05, + "loss": 5.473770556818636e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1063, + "tokens/total": 16049772, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 343645 + }, + { + "epoch": 16.625, + "grad_norm": 4.493001688388176e-05, + "learning_rate": 5.587171321780216e-05, + "loss": 1.0312560334568843e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1064, + "tokens/total": 16064916, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 343972 + }, + { + "epoch": 16.640625, + "grad_norm": 6.266077980399132e-05, + "learning_rate": 5.579907843305212e-05, + "loss": 1.023145273393311e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1065, + "tokens/total": 16080152, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 344299 + }, + { + "epoch": 16.65625, + "grad_norm": 2.876908183679916e-05, + "learning_rate": 5.572644156571396e-05, + "loss": 7.755440947221359e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1066, + "tokens/total": 16095196, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 344624 + }, + { + "epoch": 16.671875, + "grad_norm": 2.2205776986083947e-05, + "learning_rate": 5.565380280509659e-05, + "loss": 5.904433919567964e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1067, + "tokens/total": 16110188, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 344948 + }, + { + "epoch": 16.6875, + "grad_norm": 3.245476909796707e-05, + "learning_rate": 5.558116234051387e-05, + "loss": 8.371641797566554e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1068, + "tokens/total": 16125300, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 345270 + }, + { + "epoch": 16.703125, + "grad_norm": 6.63803075440228e-05, + "learning_rate": 5.55085203612841e-05, + "loss": 1.2704394976026379e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1069, + "tokens/total": 16140324, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 345595 + }, + { + "epoch": 16.71875, + "grad_norm": 8.964062726590782e-05, + "learning_rate": 5.543587705672952e-05, + "loss": 1.3437633015200845e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1070, + "tokens/total": 16155484, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 345921 + }, + { + "epoch": 16.734375, + "grad_norm": 0.00011032303882529959, + "learning_rate": 5.5363232616175845e-05, + "loss": 1.7779454992705723e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1071, + "tokens/total": 16170692, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 346250 + }, + { + "epoch": 16.75, + "grad_norm": 0.00010467788524692878, + "learning_rate": 5.529058722895172e-05, + "loss": 1.8368331211604527e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1072, + "tokens/total": 16185756, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 346575 + }, + { + "epoch": 16.765625, + "grad_norm": 8.164245809894055e-05, + "learning_rate": 5.521794108438828e-05, + "loss": 1.0885527217396884e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1073, + "tokens/total": 16200856, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 346897 + }, + { + "epoch": 16.78125, + "grad_norm": 3.503928382997401e-05, + "learning_rate": 5.5145294371818637e-05, + "loss": 8.721814879208978e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1074, + "tokens/total": 16215896, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 347221 + }, + { + "epoch": 16.796875, + "grad_norm": 3.0097548005869612e-05, + "learning_rate": 5.507264728057734e-05, + "loss": 7.06567220731813e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1075, + "tokens/total": 16231020, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 347544 + }, + { + "epoch": 16.8125, + "grad_norm": 6.028322604834102e-05, + "learning_rate": 5.500000000000001e-05, + "loss": 1.044795226334827e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1076, + "tokens/total": 16246048, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 347865 + }, + { + "epoch": 16.828125, + "grad_norm": 6.18916965322569e-05, + "learning_rate": 5.492735271942265e-05, + "loss": 1.4339573226607172e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1077, + "tokens/total": 16261216, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 348192 + }, + { + "epoch": 16.84375, + "grad_norm": 3.430634387768805e-05, + "learning_rate": 5.485470562818139e-05, + "loss": 9.710033737064805e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1078, + "tokens/total": 16276292, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 348512 + }, + { + "epoch": 16.859375, + "grad_norm": 6.602905341424048e-05, + "learning_rate": 5.4782058915611733e-05, + "loss": 1.6672578340148902e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1079, + "tokens/total": 16291260, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 348829 + }, + { + "epoch": 16.875, + "grad_norm": 4.757051283377223e-05, + "learning_rate": 5.4709412771048286e-05, + "loss": 4.669861368711281e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1080, + "tokens/total": 16306500, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 349155 + }, + { + "epoch": 16.890625, + "grad_norm": 1.853706271504052e-05, + "learning_rate": 5.463676738382416e-05, + "loss": 4.293277697797748e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1081, + "tokens/total": 16321600, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 349477 + }, + { + "epoch": 16.90625, + "grad_norm": 6.477189890574664e-05, + "learning_rate": 5.45641229432705e-05, + "loss": 1.2242967386555392e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1082, + "tokens/total": 16336680, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 349798 + }, + { + "epoch": 16.921875, + "grad_norm": 2.97890292131342e-05, + "learning_rate": 5.449147963871593e-05, + "loss": 6.905418103997363e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1083, + "tokens/total": 16351764, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 350121 + }, + { + "epoch": 16.9375, + "grad_norm": 1.6719168343115598e-05, + "learning_rate": 5.441883765948615e-05, + "loss": 3.859930757244001e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1084, + "tokens/total": 16366760, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 350441 + }, + { + "epoch": 16.953125, + "grad_norm": 5.1801162044284865e-05, + "learning_rate": 5.434619719490342e-05, + "loss": 9.675654837337788e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1085, + "tokens/total": 16381844, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 350763 + }, + { + "epoch": 16.96875, + "grad_norm": 2.2052572603570297e-05, + "learning_rate": 5.4273558434286064e-05, + "loss": 5.887176257601823e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1086, + "tokens/total": 16396936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 351083 + }, + { + "epoch": 16.984375, + "grad_norm": 4.4220429117558524e-05, + "learning_rate": 5.420092156694789e-05, + "loss": 1.1062841167586157e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1087, + "tokens/total": 16411896, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 351404 + }, + { + "epoch": 17.0, + "grad_norm": 0.00031240968382917345, + "learning_rate": 5.412828678219786e-05, + "loss": 3.3564510886208154e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1088, + "tokens/total": 16427004, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 351730 + }, + { + "epoch": 17.015625, + "grad_norm": 2.2881547920405865e-05, + "learning_rate": 5.405565426933944e-05, + "loss": 5.853499942531926e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1089, + "tokens/total": 16442060, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 352055 + }, + { + "epoch": 17.03125, + "grad_norm": 6.131173722678795e-05, + "learning_rate": 5.398302421767024e-05, + "loss": 1.4588591739084222e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1090, + "tokens/total": 16457100, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 352372 + }, + { + "epoch": 17.046875, + "grad_norm": 3.327698141220026e-05, + "learning_rate": 5.391039681648138e-05, + "loss": 8.053143574215937e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1091, + "tokens/total": 16472180, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 352692 + }, + { + "epoch": 17.0625, + "grad_norm": 4.1975035856012255e-05, + "learning_rate": 5.383777225505713e-05, + "loss": 8.537285793863703e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1092, + "tokens/total": 16487360, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 353021 + }, + { + "epoch": 17.078125, + "grad_norm": 3.938899317290634e-05, + "learning_rate": 5.376515072267434e-05, + "loss": 6.710072852911253e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1093, + "tokens/total": 16502416, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 353344 + }, + { + "epoch": 17.09375, + "grad_norm": 4.394740972202271e-05, + "learning_rate": 5.369253240860198e-05, + "loss": 1.1760636198232532e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1094, + "tokens/total": 16517468, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 353665 + }, + { + "epoch": 17.109375, + "grad_norm": 2.944147308880929e-05, + "learning_rate": 5.3619917502100604e-05, + "loss": 7.13936856300279e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1095, + "tokens/total": 16532616, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 353989 + }, + { + "epoch": 17.125, + "grad_norm": 3.705342533066869e-05, + "learning_rate": 5.3547306192421895e-05, + "loss": 7.794039902364602e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1096, + "tokens/total": 16547648, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 354315 + }, + { + "epoch": 17.140625, + "grad_norm": 4.133336551603861e-05, + "learning_rate": 5.3474698668808175e-05, + "loss": 7.316852475014457e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1097, + "tokens/total": 16562856, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 354637 + }, + { + "epoch": 17.15625, + "grad_norm": 2.7551337552722543e-05, + "learning_rate": 5.34020951204919e-05, + "loss": 5.853867151017766e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1098, + "tokens/total": 16577892, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 354957 + }, + { + "epoch": 17.171875, + "grad_norm": 3.245604966650717e-05, + "learning_rate": 5.332949573669515e-05, + "loss": 6.139665060800326e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1099, + "tokens/total": 16592988, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 355279 + }, + { + "epoch": 17.1875, + "grad_norm": 6.499417213490233e-05, + "learning_rate": 5.3256900706629133e-05, + "loss": 1.5264081412169617e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1100, + "tokens/total": 16608128, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 355603 + }, + { + "epoch": 17.203125, + "grad_norm": 4.240282578393817e-05, + "learning_rate": 5.318431021949376e-05, + "loss": 1.193069124383328e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1101, + "tokens/total": 16623244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 355925 + }, + { + "epoch": 17.21875, + "grad_norm": 1.9515131498337723e-05, + "learning_rate": 5.3111724464477084e-05, + "loss": 4.6165351363924856e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1102, + "tokens/total": 16638224, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 356247 + }, + { + "epoch": 17.234375, + "grad_norm": 2.7323492759023793e-05, + "learning_rate": 5.3039143630754796e-05, + "loss": 7.149269549699966e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1103, + "tokens/total": 16653332, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 356571 + }, + { + "epoch": 17.25, + "grad_norm": 5.23888083989732e-05, + "learning_rate": 5.2966567907489794e-05, + "loss": 1.1650665783236036e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1104, + "tokens/total": 16668436, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 356893 + }, + { + "epoch": 17.265625, + "grad_norm": 3.775634468183853e-05, + "learning_rate": 5.2893997483831634e-05, + "loss": 8.02373506303411e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1105, + "tokens/total": 16683560, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 357216 + }, + { + "epoch": 17.28125, + "grad_norm": 4.7318095312220976e-05, + "learning_rate": 5.282143254891609e-05, + "loss": 1.2687719390669372e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1106, + "tokens/total": 16698772, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 357538 + }, + { + "epoch": 17.296875, + "grad_norm": 0.00011273001291556284, + "learning_rate": 5.274887329186459e-05, + "loss": 1.8167869484386756e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1107, + "tokens/total": 16713868, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 357862 + }, + { + "epoch": 17.3125, + "grad_norm": 4.094827818335034e-05, + "learning_rate": 5.267631990178379e-05, + "loss": 1.0479004686203552e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1108, + "tokens/total": 16729000, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 358187 + }, + { + "epoch": 17.328125, + "grad_norm": 2.0659750589402393e-05, + "learning_rate": 5.260377256776506e-05, + "loss": 5.107685865368694e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1109, + "tokens/total": 16743964, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 358501 + }, + { + "epoch": 17.34375, + "grad_norm": 0.0001551241584820673, + "learning_rate": 5.2531231478884e-05, + "loss": 2.575668531790143e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1110, + "tokens/total": 16759096, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 358827 + }, + { + "epoch": 17.359375, + "grad_norm": 9.599650365998968e-05, + "learning_rate": 5.245869682419986e-05, + "loss": 8.87299279384024e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1111, + "tokens/total": 16774104, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 359154 + }, + { + "epoch": 17.375, + "grad_norm": 5.986001997371204e-05, + "learning_rate": 5.238616879275521e-05, + "loss": 1.6330698144884082e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1112, + "tokens/total": 16789204, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 359476 + }, + { + "epoch": 17.390625, + "grad_norm": 2.77586150332354e-05, + "learning_rate": 5.231364757357532e-05, + "loss": 7.522045279984013e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1113, + "tokens/total": 16804280, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 359799 + }, + { + "epoch": 17.40625, + "grad_norm": 3.6261997593101114e-05, + "learning_rate": 5.224113335566769e-05, + "loss": 9.069580642062647e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1114, + "tokens/total": 16819244, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 360121 + }, + { + "epoch": 17.421875, + "grad_norm": 2.4674765882082283e-05, + "learning_rate": 5.21686263280216e-05, + "loss": 5.104594151816855e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1115, + "tokens/total": 16834324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 360442 + }, + { + "epoch": 17.4375, + "grad_norm": 0.00010447445674799383, + "learning_rate": 5.209612667960757e-05, + "loss": 2.090980160573963e-06, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1116, + "tokens/total": 16849412, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 360765 + }, + { + "epoch": 17.453125, + "grad_norm": 3.7985038943588734e-05, + "learning_rate": 5.2023634599376926e-05, + "loss": 6.865431600999727e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1117, + "tokens/total": 16864408, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 361097 + }, + { + "epoch": 17.46875, + "grad_norm": 4.4100943341618404e-05, + "learning_rate": 5.195115027626119e-05, + "loss": 8.716349384485511e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1118, + "tokens/total": 16879600, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 361419 + }, + { + "epoch": 17.484375, + "grad_norm": 1.803486520657316e-05, + "learning_rate": 5.187867389917177e-05, + "loss": 4.183801820545341e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1119, + "tokens/total": 16894664, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 361744 + }, + { + "epoch": 17.5, + "grad_norm": 5.7857461797539145e-05, + "learning_rate": 5.1806205656999295e-05, + "loss": 7.998127102837316e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1120, + "tokens/total": 16909820, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 362068 + }, + { + "epoch": 17.515625, + "grad_norm": 2.8921769626322202e-05, + "learning_rate": 5.173374573861322e-05, + "loss": 6.113073141023051e-07, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1121, + "tokens/total": 16924896, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 362389 + }, + { + "epoch": 17.53125, + "grad_norm": 0.0002817696949932724, + "learning_rate": 5.166129433286125e-05, + "loss": 2.827458501997171e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.02, + "memory/max_allocated (GiB)": 33.02, + "ppl": 1.0, + "step": 1122, + "tokens/total": 16940088, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 362709 + }, + { + "epoch": 17.546875, + "grad_norm": 2.8500400730990805e-05, + "learning_rate": 5.1588851628569024e-05, + "loss": 6.019486704644805e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1123, + "tokens/total": 16955248, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 363033 + }, + { + "epoch": 17.5625, + "grad_norm": 4.83815892948769e-05, + "learning_rate": 5.151641781453936e-05, + "loss": 7.401708330689871e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1124, + "tokens/total": 16970420, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 363357 + }, + { + "epoch": 17.578125, + "grad_norm": 2.7673559088725597e-05, + "learning_rate": 5.1443993079552045e-05, + "loss": 5.301401984070253e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1125, + "tokens/total": 16985544, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 363681 + }, + { + "epoch": 17.59375, + "grad_norm": 2.9768614695058204e-05, + "learning_rate": 5.137157761236308e-05, + "loss": 7.340085517171246e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1126, + "tokens/total": 17000660, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 364005 + }, + { + "epoch": 17.609375, + "grad_norm": 2.341053004784044e-05, + "learning_rate": 5.129917160170439e-05, + "loss": 6.658422648797568e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1127, + "tokens/total": 17015752, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 364331 + }, + { + "epoch": 17.625, + "grad_norm": 2.2069334590923972e-05, + "learning_rate": 5.122677523628321e-05, + "loss": 5.573359658228583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1128, + "tokens/total": 17030796, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 364655 + }, + { + "epoch": 17.640625, + "grad_norm": 2.1251438738545403e-05, + "learning_rate": 5.1154388704781705e-05, + "loss": 5.52477956716757e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1129, + "tokens/total": 17045880, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 364980 + }, + { + "epoch": 17.65625, + "grad_norm": 4.1272516682511196e-05, + "learning_rate": 5.108201219585631e-05, + "loss": 7.258950063260272e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1130, + "tokens/total": 17061068, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 365303 + }, + { + "epoch": 17.671875, + "grad_norm": 4.5920951379230246e-05, + "learning_rate": 5.100964589813742e-05, + "loss": 4.244865863256564e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1131, + "tokens/total": 17076200, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 365624 + }, + { + "epoch": 17.6875, + "grad_norm": 6.216612382559106e-05, + "learning_rate": 5.093729000022879e-05, + "loss": 9.887139640341047e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1132, + "tokens/total": 17091204, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 365948 + }, + { + "epoch": 17.703125, + "grad_norm": 2.120691533491481e-05, + "learning_rate": 5.0864944690707076e-05, + "loss": 3.9635062876186566e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1133, + "tokens/total": 17106228, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 366273 + }, + { + "epoch": 17.71875, + "grad_norm": 2.438397496007383e-05, + "learning_rate": 5.079261015812131e-05, + "loss": 5.817321380163776e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1134, + "tokens/total": 17121480, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 366591 + }, + { + "epoch": 17.734375, + "grad_norm": 1.571947905176785e-05, + "learning_rate": 5.072028659099246e-05, + "loss": 3.932775030079938e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1135, + "tokens/total": 17136436, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 366912 + }, + { + "epoch": 17.75, + "grad_norm": 3.359025140525773e-05, + "learning_rate": 5.0647974177812926e-05, + "loss": 7.664742724955431e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1136, + "tokens/total": 17151492, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 367238 + }, + { + "epoch": 17.765625, + "grad_norm": 3.316184665891342e-05, + "learning_rate": 5.057567310704605e-05, + "loss": 8.991078743747494e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1137, + "tokens/total": 17166708, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 367560 + }, + { + "epoch": 17.78125, + "grad_norm": 6.0858172219013795e-05, + "learning_rate": 5.050338356712555e-05, + "loss": 8.310257157972956e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1138, + "tokens/total": 17181856, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 367887 + }, + { + "epoch": 17.796875, + "grad_norm": 6.144674262031913e-05, + "learning_rate": 5.043110574645514e-05, + "loss": 1.0170548421228887e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1139, + "tokens/total": 17196996, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 368209 + }, + { + "epoch": 17.8125, + "grad_norm": 5.4178250138647854e-05, + "learning_rate": 5.0358839833408e-05, + "loss": 8.161293862940511e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1140, + "tokens/total": 17212124, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 368535 + }, + { + "epoch": 17.828125, + "grad_norm": 0.00027458579279482365, + "learning_rate": 5.0286586016326244e-05, + "loss": 2.3955503820616286e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1141, + "tokens/total": 17227176, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 368854 + }, + { + "epoch": 17.84375, + "grad_norm": 1.4002062016515993e-05, + "learning_rate": 5.021434448352047e-05, + "loss": 3.361631684128952e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1142, + "tokens/total": 17242328, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 369179 + }, + { + "epoch": 17.859375, + "grad_norm": 7.755967817502096e-05, + "learning_rate": 5.014211542326928e-05, + "loss": 8.134556424010952e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1143, + "tokens/total": 17257444, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 369503 + }, + { + "epoch": 17.875, + "grad_norm": 3.116878360742703e-05, + "learning_rate": 5.006989902381873e-05, + "loss": 6.312773734862276e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1144, + "tokens/total": 17272628, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 369826 + }, + { + "epoch": 17.890625, + "grad_norm": 2.3439159122062847e-05, + "learning_rate": 4.999769547338196e-05, + "loss": 6.016841211931023e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1145, + "tokens/total": 17287644, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 370157 + }, + { + "epoch": 17.90625, + "grad_norm": 3.5790970287052915e-05, + "learning_rate": 4.99255049601385e-05, + "loss": 8.734820085010142e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1146, + "tokens/total": 17302800, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 370481 + }, + { + "epoch": 17.921875, + "grad_norm": 2.4421453417744488e-05, + "learning_rate": 4.9853327672234e-05, + "loss": 4.3543334982132365e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1147, + "tokens/total": 17317872, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 370806 + }, + { + "epoch": 17.9375, + "grad_norm": 4.263213122612797e-05, + "learning_rate": 4.97811637977796e-05, + "loss": 6.137996706456761e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1148, + "tokens/total": 17332960, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 371131 + }, + { + "epoch": 17.953125, + "grad_norm": 5.632291504298337e-05, + "learning_rate": 4.970901352485152e-05, + "loss": 9.398837619301048e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1149, + "tokens/total": 17348028, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 371455 + }, + { + "epoch": 17.96875, + "grad_norm": 2.0084356947336346e-05, + "learning_rate": 4.963687704149047e-05, + "loss": 4.643854936148273e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1150, + "tokens/total": 17363192, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 371777 + }, + { + "epoch": 17.984375, + "grad_norm": 8.160856668837368e-05, + "learning_rate": 4.9564754535701255e-05, + "loss": 1.5316568351408932e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1151, + "tokens/total": 17378380, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 372097 + }, + { + "epoch": 18.0, + "grad_norm": 3.531977563397959e-05, + "learning_rate": 4.9492646195452284e-05, + "loss": 7.746496066829422e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1152, + "tokens/total": 17393556, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 372420 + }, + { + "epoch": 18.015625, + "grad_norm": 1.9974901078967378e-05, + "learning_rate": 4.9420552208674964e-05, + "loss": 5.18842455221602e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1153, + "tokens/total": 17408624, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 372740 + }, + { + "epoch": 18.03125, + "grad_norm": 6.941713218111545e-05, + "learning_rate": 4.934847276326337e-05, + "loss": 9.98698737930681e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1154, + "tokens/total": 17423700, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 373062 + }, + { + "epoch": 18.046875, + "grad_norm": 4.135937342653051e-05, + "learning_rate": 4.927640804707364e-05, + "loss": 8.435266636297456e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1155, + "tokens/total": 17438852, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 373385 + }, + { + "epoch": 18.0625, + "grad_norm": 4.7824898501858115e-05, + "learning_rate": 4.9204358247923573e-05, + "loss": 1.0068520168715622e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1156, + "tokens/total": 17453868, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 373709 + }, + { + "epoch": 18.078125, + "grad_norm": 3.0483102818834595e-05, + "learning_rate": 4.913232355359201e-05, + "loss": 7.162204838095931e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1157, + "tokens/total": 17468880, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 374029 + }, + { + "epoch": 18.09375, + "grad_norm": 2.470616345817689e-05, + "learning_rate": 4.906030415181847e-05, + "loss": 6.548955866492179e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1158, + "tokens/total": 17483868, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 374353 + }, + { + "epoch": 18.109375, + "grad_norm": 3.0207020245143212e-05, + "learning_rate": 4.8988300230302634e-05, + "loss": 7.646083304280182e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1159, + "tokens/total": 17499032, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 374679 + }, + { + "epoch": 18.125, + "grad_norm": 3.6233061109669507e-05, + "learning_rate": 4.891631197670381e-05, + "loss": 5.78564140596427e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1160, + "tokens/total": 17514008, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 375000 + }, + { + "epoch": 18.140625, + "grad_norm": 4.496629844652489e-05, + "learning_rate": 4.8844339578640484e-05, + "loss": 7.683796070523385e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1161, + "tokens/total": 17529212, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 375325 + }, + { + "epoch": 18.15625, + "grad_norm": 1.7914642739924602e-05, + "learning_rate": 4.87723832236898e-05, + "loss": 4.116704701573326e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1162, + "tokens/total": 17544304, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 375650 + }, + { + "epoch": 18.171875, + "grad_norm": 1.719811189104803e-05, + "learning_rate": 4.8700443099387115e-05, + "loss": 4.2161005353591463e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1163, + "tokens/total": 17559500, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 375974 + }, + { + "epoch": 18.1875, + "grad_norm": 6.385466258507222e-05, + "learning_rate": 4.8628519393225494e-05, + "loss": 1.3081686347504728e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1164, + "tokens/total": 17574504, + "tokens/train_per_sec_per_gpu": 8.91, + "tokens/trainable": 376294 + }, + { + "epoch": 18.203125, + "grad_norm": 5.1405215344857424e-05, + "learning_rate": 4.855661229265516e-05, + "loss": 5.563985041590058e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1165, + "tokens/total": 17589612, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 376614 + }, + { + "epoch": 18.21875, + "grad_norm": 3.782465500989929e-05, + "learning_rate": 4.848472198508308e-05, + "loss": 7.833817790015019e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1166, + "tokens/total": 17604728, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 376937 + }, + { + "epoch": 18.234375, + "grad_norm": 2.4382250558119267e-05, + "learning_rate": 4.84128486578725e-05, + "loss": 6.255218067963142e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1167, + "tokens/total": 17619828, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 377262 + }, + { + "epoch": 18.25, + "grad_norm": 2.2193116819835268e-05, + "learning_rate": 4.834099249834238e-05, + "loss": 5.302445060806349e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1168, + "tokens/total": 17634884, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 377581 + }, + { + "epoch": 18.265625, + "grad_norm": 4.160241951467469e-05, + "learning_rate": 4.826915369376691e-05, + "loss": 7.34506215849251e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1169, + "tokens/total": 17650028, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 377906 + }, + { + "epoch": 18.28125, + "grad_norm": 1.5566462025162764e-05, + "learning_rate": 4.819733243137508e-05, + "loss": 3.547885683019558e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1170, + "tokens/total": 17665088, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 378228 + }, + { + "epoch": 18.296875, + "grad_norm": 2.2747379262000322e-05, + "learning_rate": 4.8125528898350135e-05, + "loss": 5.782691232525394e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1171, + "tokens/total": 17680180, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 378551 + }, + { + "epoch": 18.3125, + "grad_norm": 0.00023228925419971347, + "learning_rate": 4.805374328182918e-05, + "loss": 2.2927927147975424e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1172, + "tokens/total": 17695212, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 378879 + }, + { + "epoch": 18.328125, + "grad_norm": 3.2566960726398975e-05, + "learning_rate": 4.798197576890252e-05, + "loss": 7.130024073376262e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1173, + "tokens/total": 17710364, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 379205 + }, + { + "epoch": 18.34375, + "grad_norm": 4.793679545400664e-05, + "learning_rate": 4.791022654661336e-05, + "loss": 1.2894919336758903e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.0, + "step": 1174, + "tokens/total": 17725380, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 379530 + }, + { + "epoch": 18.359375, + "grad_norm": 2.17981305468129e-05, + "learning_rate": 4.7838495801957214e-05, + "loss": 4.1462456579210993e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1175, + "tokens/total": 17740320, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 379855 + }, + { + "epoch": 18.375, + "grad_norm": 1.93180294445483e-05, + "learning_rate": 4.7766783721881436e-05, + "loss": 4.267131998858531e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1176, + "tokens/total": 17755380, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 380186 + }, + { + "epoch": 18.390625, + "grad_norm": 2.5789411665755324e-05, + "learning_rate": 4.7695090493284724e-05, + "loss": 5.850677666785487e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1177, + "tokens/total": 17770508, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 380511 + }, + { + "epoch": 18.40625, + "grad_norm": 2.7064219466410577e-05, + "learning_rate": 4.762341630301666e-05, + "loss": 6.367350806613103e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1178, + "tokens/total": 17785644, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 380831 + }, + { + "epoch": 18.421875, + "grad_norm": 3.8992824556771666e-05, + "learning_rate": 4.7551761337877175e-05, + "loss": 8.640238888801832e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1179, + "tokens/total": 17800712, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 381153 + }, + { + "epoch": 18.4375, + "grad_norm": 4.346343484940007e-05, + "learning_rate": 4.748012578461618e-05, + "loss": 7.757635671623575e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1180, + "tokens/total": 17815812, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 381475 + }, + { + "epoch": 18.453125, + "grad_norm": 1.4182709492160939e-05, + "learning_rate": 4.740850982993288e-05, + "loss": 3.3511992114654277e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1181, + "tokens/total": 17830848, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 381798 + }, + { + "epoch": 18.46875, + "grad_norm": 4.446584716788493e-05, + "learning_rate": 4.733691366047548e-05, + "loss": 8.970853855316818e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1182, + "tokens/total": 17845968, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 382119 + }, + { + "epoch": 18.484375, + "grad_norm": 2.0205559849273413e-05, + "learning_rate": 4.72653374628406e-05, + "loss": 4.501660555433773e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1183, + "tokens/total": 17861116, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 382443 + }, + { + "epoch": 18.5, + "grad_norm": 4.77209068776574e-05, + "learning_rate": 4.7193781423572815e-05, + "loss": 7.883093076088699e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1184, + "tokens/total": 17876312, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 382767 + }, + { + "epoch": 18.515625, + "grad_norm": 9.563214553054422e-05, + "learning_rate": 4.7122245729164115e-05, + "loss": 2.184342292821384e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1185, + "tokens/total": 17891544, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 383090 + }, + { + "epoch": 18.53125, + "grad_norm": 3.0818577215541154e-05, + "learning_rate": 4.705073056605355e-05, + "loss": 7.60321256620955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1186, + "tokens/total": 17906680, + "tokens/train_per_sec_per_gpu": 10.2, + "tokens/trainable": 383418 + }, + { + "epoch": 18.546875, + "grad_norm": 2.7632315322989598e-05, + "learning_rate": 4.697923612062659e-05, + "loss": 7.209595196400187e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1187, + "tokens/total": 17921896, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 383740 + }, + { + "epoch": 18.5625, + "grad_norm": 0.00015282968524843454, + "learning_rate": 4.6907762579214744e-05, + "loss": 2.5318040570709854e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1188, + "tokens/total": 17937040, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 384060 + }, + { + "epoch": 18.578125, + "grad_norm": 3.8941463571973145e-05, + "learning_rate": 4.6836310128095015e-05, + "loss": 7.086154028002056e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1189, + "tokens/total": 17952080, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 384381 + }, + { + "epoch": 18.59375, + "grad_norm": 3.0331184461829253e-05, + "learning_rate": 4.676487895348949e-05, + "loss": 8.000384923434467e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1190, + "tokens/total": 17967236, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 384704 + }, + { + "epoch": 18.609375, + "grad_norm": 2.89824038191e-05, + "learning_rate": 4.6693469241564755e-05, + "loss": 7.461384257112513e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1191, + "tokens/total": 17982240, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 385023 + }, + { + "epoch": 18.625, + "grad_norm": 3.457655475358479e-05, + "learning_rate": 4.662208117843145e-05, + "loss": 8.100512900455215e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1192, + "tokens/total": 17997448, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 385350 + }, + { + "epoch": 18.640625, + "grad_norm": 2.2200512830750085e-05, + "learning_rate": 4.6550714950143846e-05, + "loss": 5.453988478620886e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1193, + "tokens/total": 18012444, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 385676 + }, + { + "epoch": 18.65625, + "grad_norm": 3.124560316791758e-05, + "learning_rate": 4.6479370742699277e-05, + "loss": 7.036871920718113e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1194, + "tokens/total": 18027620, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 386000 + }, + { + "epoch": 18.671875, + "grad_norm": 1.5807234376552515e-05, + "learning_rate": 4.640804874203769e-05, + "loss": 2.390911504335236e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1195, + "tokens/total": 18042704, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 386324 + }, + { + "epoch": 18.6875, + "grad_norm": 2.4668333935551345e-05, + "learning_rate": 4.633674913404112e-05, + "loss": 6.089125008656993e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1196, + "tokens/total": 18057756, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 386648 + }, + { + "epoch": 18.703125, + "grad_norm": 1.4880516573612113e-05, + "learning_rate": 4.626547210453333e-05, + "loss": 3.5727714475797256e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.01, + "memory/max_allocated (GiB)": 33.01, + "ppl": 1.0, + "step": 1197, + "tokens/total": 18072840, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 386971 + }, + { + "epoch": 18.71875, + "grad_norm": 3.726745489984751e-05, + "learning_rate": 4.619421783927913e-05, + "loss": 6.179744218570704e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1198, + "tokens/total": 18087852, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 387292 + }, + { + "epoch": 18.734375, + "grad_norm": 6.82832978782244e-05, + "learning_rate": 4.61229865239841e-05, + "loss": 1.35417474211863e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1199, + "tokens/total": 18102908, + "tokens/train_per_sec_per_gpu": 10.07, + "tokens/trainable": 387617 + }, + { + "epoch": 18.75, + "grad_norm": 3.162693974445574e-05, + "learning_rate": 4.6051778344293914e-05, + "loss": 7.62121374009439e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1200, + "tokens/total": 18117924, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 387939 + }, + { + "epoch": 18.765625, + "grad_norm": 2.133204907295294e-05, + "learning_rate": 4.598059348579402e-05, + "loss": 4.981645815860247e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1201, + "tokens/total": 18133012, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 388262 + }, + { + "epoch": 18.78125, + "grad_norm": 2.5610261218389496e-05, + "learning_rate": 4.590943213400907e-05, + "loss": 6.275737405303516e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1202, + "tokens/total": 18148140, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 388585 + }, + { + "epoch": 18.796875, + "grad_norm": 3.05334287986625e-05, + "learning_rate": 4.583829447440244e-05, + "loss": 5.718101760976424e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1203, + "tokens/total": 18163272, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 388910 + }, + { + "epoch": 18.8125, + "grad_norm": 4.8873229388846084e-05, + "learning_rate": 4.576718069237575e-05, + "loss": 5.625361154670827e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1204, + "tokens/total": 18178428, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 389238 + }, + { + "epoch": 18.828125, + "grad_norm": 1.847824933065567e-05, + "learning_rate": 4.56960909732684e-05, + "loss": 3.800790295827028e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1205, + "tokens/total": 18193456, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 389563 + }, + { + "epoch": 18.84375, + "grad_norm": 2.659624988154974e-05, + "learning_rate": 4.562502550235709e-05, + "loss": 6.226485993465758e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1206, + "tokens/total": 18208416, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 389886 + }, + { + "epoch": 18.859375, + "grad_norm": 4.955941767548211e-05, + "learning_rate": 4.555398446485533e-05, + "loss": 7.174953680078033e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1207, + "tokens/total": 18223516, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 390207 + }, + { + "epoch": 18.875, + "grad_norm": 2.1514479158213362e-05, + "learning_rate": 4.548296804591287e-05, + "loss": 4.4104712060288875e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1208, + "tokens/total": 18238636, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 390529 + }, + { + "epoch": 18.890625, + "grad_norm": 3.941807153751142e-05, + "learning_rate": 4.541197643061541e-05, + "loss": 8.870987926457019e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1209, + "tokens/total": 18253796, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 390857 + }, + { + "epoch": 18.90625, + "grad_norm": 2.6891915695159696e-05, + "learning_rate": 4.5341009803983924e-05, + "loss": 6.029642918292666e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1210, + "tokens/total": 18268888, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 391180 + }, + { + "epoch": 18.921875, + "grad_norm": 3.027707316505257e-05, + "learning_rate": 4.527006835097435e-05, + "loss": 6.475734721789195e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1211, + "tokens/total": 18284088, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 391501 + }, + { + "epoch": 18.9375, + "grad_norm": 2.2468319002655335e-05, + "learning_rate": 4.5199152256476895e-05, + "loss": 5.615971758743399e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1212, + "tokens/total": 18299152, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 391822 + }, + { + "epoch": 18.953125, + "grad_norm": 0.0002802519884426147, + "learning_rate": 4.512826170531577e-05, + "loss": 2.8803433451685123e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1213, + "tokens/total": 18314300, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 392146 + }, + { + "epoch": 18.96875, + "grad_norm": 4.769030419993214e-05, + "learning_rate": 4.505739688224856e-05, + "loss": 4.984336214874929e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1214, + "tokens/total": 18329512, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 392470 + }, + { + "epoch": 18.984375, + "grad_norm": 1.293989953410346e-05, + "learning_rate": 4.498655797196586e-05, + "loss": 2.928850619809964e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1215, + "tokens/total": 18344580, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 392791 + }, + { + "epoch": 19.0, + "grad_norm": 4.2664571083150804e-05, + "learning_rate": 4.491574515909066e-05, + "loss": 6.677284432043962e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1216, + "tokens/total": 18359720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 393110 + }, + { + "epoch": 19.015625, + "grad_norm": 0.0002413819747744128, + "learning_rate": 4.484495862817796e-05, + "loss": 2.760701136139687e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1217, + "tokens/total": 18374876, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 393431 + }, + { + "epoch": 19.03125, + "grad_norm": 3.334211214678362e-05, + "learning_rate": 4.4774198563714274e-05, + "loss": 6.117605835243012e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1218, + "tokens/total": 18389976, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 393755 + }, + { + "epoch": 19.046875, + "grad_norm": 2.101369682350196e-05, + "learning_rate": 4.470346515011716e-05, + "loss": 4.4172716684443003e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1219, + "tokens/total": 18405156, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 394077 + }, + { + "epoch": 19.0625, + "grad_norm": 5.3968309657648206e-05, + "learning_rate": 4.4632758571734645e-05, + "loss": 6.536999990203185e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1220, + "tokens/total": 18420180, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 394397 + }, + { + "epoch": 19.078125, + "grad_norm": 4.533410901785828e-05, + "learning_rate": 4.456207901284488e-05, + "loss": 9.584194913259125e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1221, + "tokens/total": 18435268, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 394714 + }, + { + "epoch": 19.09375, + "grad_norm": 2.2877778974361718e-05, + "learning_rate": 4.44914266576556e-05, + "loss": 5.096880499877443e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1222, + "tokens/total": 18450336, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 395039 + }, + { + "epoch": 19.109375, + "grad_norm": 2.626161767693702e-05, + "learning_rate": 4.442080169030356e-05, + "loss": 5.378892637963872e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1223, + "tokens/total": 18465452, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 395362 + }, + { + "epoch": 19.125, + "grad_norm": 6.159742770250887e-05, + "learning_rate": 4.435020429485425e-05, + "loss": 1.239257471752353e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1224, + "tokens/total": 18480552, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 395690 + }, + { + "epoch": 19.140625, + "grad_norm": 1.8510498193791136e-05, + "learning_rate": 4.427963465530121e-05, + "loss": 4.4955950784242305e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1225, + "tokens/total": 18495576, + "tokens/train_per_sec_per_gpu": 8.58, + "tokens/trainable": 396014 + }, + { + "epoch": 19.15625, + "grad_norm": 5.079836773802526e-05, + "learning_rate": 4.420909295556571e-05, + "loss": 5.883057951905357e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1226, + "tokens/total": 18510732, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 396336 + }, + { + "epoch": 19.171875, + "grad_norm": 6.822404247941449e-05, + "learning_rate": 4.413857937949614e-05, + "loss": 8.770913382250001e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1227, + "tokens/total": 18525832, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 396660 + }, + { + "epoch": 19.1875, + "grad_norm": 4.7934041504049674e-05, + "learning_rate": 4.406809411086763e-05, + "loss": 7.590152790726279e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1228, + "tokens/total": 18540872, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 396983 + }, + { + "epoch": 19.203125, + "grad_norm": 2.897239028243348e-05, + "learning_rate": 4.3997637333381524e-05, + "loss": 6.361534587995266e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1229, + "tokens/total": 18555872, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 397307 + }, + { + "epoch": 19.21875, + "grad_norm": 2.1820884285261855e-05, + "learning_rate": 4.3927209230664934e-05, + "loss": 4.563100333143666e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1230, + "tokens/total": 18571032, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 397633 + }, + { + "epoch": 19.234375, + "grad_norm": 3.0124436307232827e-05, + "learning_rate": 4.385680998627019e-05, + "loss": 7.310122214221337e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1231, + "tokens/total": 18586096, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 397957 + }, + { + "epoch": 19.25, + "grad_norm": 3.377087341505103e-05, + "learning_rate": 4.378643978367445e-05, + "loss": 6.118332862570242e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1232, + "tokens/total": 18601160, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 398280 + }, + { + "epoch": 19.265625, + "grad_norm": 1.699730273685418e-05, + "learning_rate": 4.371609880627917e-05, + "loss": 3.862602113713365e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1233, + "tokens/total": 18616308, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 398604 + }, + { + "epoch": 19.28125, + "grad_norm": 5.8391509810462594e-05, + "learning_rate": 4.364578723740964e-05, + "loss": 1.0024249377238448e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1234, + "tokens/total": 18631468, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 398927 + }, + { + "epoch": 19.296875, + "grad_norm": 4.258325498085469e-05, + "learning_rate": 4.3575505260314506e-05, + "loss": 8.840386840347492e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1235, + "tokens/total": 18646552, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 399253 + }, + { + "epoch": 19.3125, + "grad_norm": 1.6840176613186486e-05, + "learning_rate": 4.3505253058165275e-05, + "loss": 3.657573302007222e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1236, + "tokens/total": 18661528, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 399575 + }, + { + "epoch": 19.328125, + "grad_norm": 1.3967413906357251e-05, + "learning_rate": 4.3435030814055876e-05, + "loss": 2.8899239623569883e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1237, + "tokens/total": 18676632, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 399900 + }, + { + "epoch": 19.34375, + "grad_norm": 1.4497553820547182e-05, + "learning_rate": 4.336483871100218e-05, + "loss": 3.351066766299482e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1238, + "tokens/total": 18691772, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 400224 + }, + { + "epoch": 19.359375, + "grad_norm": 2.5100594939431176e-05, + "learning_rate": 4.3294676931941405e-05, + "loss": 4.5932341663501575e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1239, + "tokens/total": 18706884, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 400548 + }, + { + "epoch": 19.375, + "grad_norm": 5.760142084909603e-05, + "learning_rate": 4.3224545659731854e-05, + "loss": 1.1583445029828e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1240, + "tokens/total": 18722128, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 400873 + }, + { + "epoch": 19.390625, + "grad_norm": 5.9057139878859743e-05, + "learning_rate": 4.315444507715226e-05, + "loss": 6.992169687691785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1241, + "tokens/total": 18737192, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 401199 + }, + { + "epoch": 19.40625, + "grad_norm": 4.200682451482862e-05, + "learning_rate": 4.308437536690142e-05, + "loss": 4.2311395986871503e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1242, + "tokens/total": 18752228, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 401521 + }, + { + "epoch": 19.421875, + "grad_norm": 2.1798921807203442e-05, + "learning_rate": 4.3014336711597594e-05, + "loss": 5.104765250507626e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1243, + "tokens/total": 18767256, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 401843 + }, + { + "epoch": 19.4375, + "grad_norm": 2.039364153461065e-05, + "learning_rate": 4.294432929377814e-05, + "loss": 4.5101322143636935e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1244, + "tokens/total": 18782420, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 402165 + }, + { + "epoch": 19.453125, + "grad_norm": 5.295968003338203e-05, + "learning_rate": 4.287435329589906e-05, + "loss": 3.3419098599551944e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1245, + "tokens/total": 18797532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 402488 + }, + { + "epoch": 19.46875, + "grad_norm": 1.7035577911883593e-05, + "learning_rate": 4.2804408900334394e-05, + "loss": 3.3760022688511526e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1246, + "tokens/total": 18812688, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 402813 + }, + { + "epoch": 19.484375, + "grad_norm": 1.7198868590639904e-05, + "learning_rate": 4.273449628937583e-05, + "loss": 4.1589908050809754e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1247, + "tokens/total": 18827700, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 403136 + }, + { + "epoch": 19.5, + "grad_norm": 1.4545736121363007e-05, + "learning_rate": 4.266461564523224e-05, + "loss": 3.56315837279908e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1248, + "tokens/total": 18842784, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 403463 + }, + { + "epoch": 19.515625, + "grad_norm": 1.1580904356378596e-05, + "learning_rate": 4.259476715002919e-05, + "loss": 2.5386378865732695e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1249, + "tokens/total": 18857880, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 403786 + }, + { + "epoch": 19.53125, + "grad_norm": 1.899923699966166e-05, + "learning_rate": 4.2524950985808433e-05, + "loss": 4.7036269279487897e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1250, + "tokens/total": 18873028, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 404115 + }, + { + "epoch": 19.546875, + "grad_norm": 5.232199328020215e-05, + "learning_rate": 4.2455167334527466e-05, + "loss": 9.801951819099486e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1251, + "tokens/total": 18888056, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 404440 + }, + { + "epoch": 19.5625, + "grad_norm": 0.00017999687406700104, + "learning_rate": 4.238541637805905e-05, + "loss": 1.8908290257968474e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1252, + "tokens/total": 18903208, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 404765 + }, + { + "epoch": 19.578125, + "grad_norm": 1.7974878574023023e-05, + "learning_rate": 4.231569829819075e-05, + "loss": 4.176444292625092e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1253, + "tokens/total": 18918404, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 405087 + }, + { + "epoch": 19.59375, + "grad_norm": 1.56655896716984e-05, + "learning_rate": 4.2246013276624464e-05, + "loss": 3.4921663427667227e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1254, + "tokens/total": 18933532, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 405407 + }, + { + "epoch": 19.609375, + "grad_norm": 3.995351289631799e-05, + "learning_rate": 4.217636149497584e-05, + "loss": 9.20505158319429e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1255, + "tokens/total": 18948644, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 405727 + }, + { + "epoch": 19.625, + "grad_norm": 1.4716701116412878e-05, + "learning_rate": 4.210674313477399e-05, + "loss": 3.31877487269594e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1256, + "tokens/total": 18963700, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 406054 + }, + { + "epoch": 19.640625, + "grad_norm": 2.6169722332269885e-05, + "learning_rate": 4.203715837746092e-05, + "loss": 6.605737326026428e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1257, + "tokens/total": 18978812, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 406374 + }, + { + "epoch": 19.65625, + "grad_norm": 4.2985597247025e-05, + "learning_rate": 4.196760740439097e-05, + "loss": 6.935694045751006e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1258, + "tokens/total": 18993824, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 406696 + }, + { + "epoch": 19.671875, + "grad_norm": 2.071825656457804e-05, + "learning_rate": 4.189809039683055e-05, + "loss": 5.028953182772966e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1259, + "tokens/total": 19009044, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 407018 + }, + { + "epoch": 19.6875, + "grad_norm": 1.8332286344957538e-05, + "learning_rate": 4.1828607535957435e-05, + "loss": 4.4753858219337417e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1260, + "tokens/total": 19024196, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 407344 + }, + { + "epoch": 19.703125, + "grad_norm": 2.3981798221939243e-05, + "learning_rate": 4.1759159002860504e-05, + "loss": 4.300611067264981e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1261, + "tokens/total": 19039252, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 407668 + }, + { + "epoch": 19.71875, + "grad_norm": 2.897316517191939e-05, + "learning_rate": 4.168974497853909e-05, + "loss": 7.343402899095963e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1262, + "tokens/total": 19054368, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 407991 + }, + { + "epoch": 19.734375, + "grad_norm": 2.8219616069691256e-05, + "learning_rate": 4.162036564390264e-05, + "loss": 6.682337243546499e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1263, + "tokens/total": 19069548, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 408313 + }, + { + "epoch": 19.75, + "grad_norm": 1.9494475054671057e-05, + "learning_rate": 4.1551021179770166e-05, + "loss": 4.917721980746137e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1264, + "tokens/total": 19084724, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 408635 + }, + { + "epoch": 19.765625, + "grad_norm": 4.063935193698853e-05, + "learning_rate": 4.148171176686983e-05, + "loss": 6.950397732907732e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1265, + "tokens/total": 19099812, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 408957 + }, + { + "epoch": 19.78125, + "grad_norm": 0.00011342860670993105, + "learning_rate": 4.141243758583837e-05, + "loss": 1.8526593521528412e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1266, + "tokens/total": 19114956, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 409276 + }, + { + "epoch": 19.796875, + "grad_norm": 2.754404158622492e-05, + "learning_rate": 4.1343198817220784e-05, + "loss": 4.871458259003703e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1267, + "tokens/total": 19130024, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 409599 + }, + { + "epoch": 19.8125, + "grad_norm": 4.316239937907085e-05, + "learning_rate": 4.127399564146975e-05, + "loss": 9.459103011977277e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1268, + "tokens/total": 19145196, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 409923 + }, + { + "epoch": 19.828125, + "grad_norm": 4.4577809603651986e-05, + "learning_rate": 4.120482823894516e-05, + "loss": 1.021559114633419e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1269, + "tokens/total": 19160252, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 410246 + }, + { + "epoch": 19.84375, + "grad_norm": 2.5966159228119068e-05, + "learning_rate": 4.113569678991369e-05, + "loss": 4.852566917179502e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1270, + "tokens/total": 19175352, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 410571 + }, + { + "epoch": 19.859375, + "grad_norm": 3.940537862945348e-05, + "learning_rate": 4.106660147454832e-05, + "loss": 7.648293944839679e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1271, + "tokens/total": 19190464, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 410890 + }, + { + "epoch": 19.875, + "grad_norm": 1.1024420928151812e-05, + "learning_rate": 4.099754247292782e-05, + "loss": 2.626149750994955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1272, + "tokens/total": 19205592, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 411217 + }, + { + "epoch": 19.890625, + "grad_norm": 2.8593740353244357e-05, + "learning_rate": 4.0928519965036385e-05, + "loss": 6.371573135766084e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1273, + "tokens/total": 19220660, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 411541 + }, + { + "epoch": 19.90625, + "grad_norm": 3.38174395437818e-05, + "learning_rate": 4.085953413076302e-05, + "loss": 5.64643244160834e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1274, + "tokens/total": 19235708, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 411863 + }, + { + "epoch": 19.921875, + "grad_norm": 2.0192017473164015e-05, + "learning_rate": 4.079058514990122e-05, + "loss": 4.5727509245807596e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1275, + "tokens/total": 19250720, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 412184 + }, + { + "epoch": 19.9375, + "grad_norm": 1.8245375031256117e-05, + "learning_rate": 4.072167320214838e-05, + "loss": 4.452640496310778e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1276, + "tokens/total": 19265824, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 412505 + }, + { + "epoch": 19.953125, + "grad_norm": 2.8479164029704407e-05, + "learning_rate": 4.0652798467105425e-05, + "loss": 5.579213393502869e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1277, + "tokens/total": 19280920, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 412831 + }, + { + "epoch": 19.96875, + "grad_norm": 2.4118136934703216e-05, + "learning_rate": 4.058396112427622e-05, + "loss": 4.416513093019603e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1278, + "tokens/total": 19295996, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 413157 + }, + { + "epoch": 19.984375, + "grad_norm": 1.949789293576032e-05, + "learning_rate": 4.0515161353067274e-05, + "loss": 4.5959188810229534e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1279, + "tokens/total": 19311200, + "tokens/train_per_sec_per_gpu": 8.57, + "tokens/trainable": 413475 + }, + { + "epoch": 20.0, + "grad_norm": 1.787384826457128e-05, + "learning_rate": 4.04463993327871e-05, + "loss": 3.9502589288531453e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1280, + "tokens/total": 19326268, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 413800 + }, + { + "epoch": 20.015625, + "grad_norm": 0.00015468464698642492, + "learning_rate": 4.037767524264587e-05, + "loss": 1.6412000150012318e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1281, + "tokens/total": 19341372, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 414124 + }, + { + "epoch": 20.03125, + "grad_norm": 3.30606235365849e-05, + "learning_rate": 4.030898926175486e-05, + "loss": 6.44645183456305e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1282, + "tokens/total": 19356556, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 414449 + }, + { + "epoch": 20.046875, + "grad_norm": 2.100896745105274e-05, + "learning_rate": 4.024034156912605e-05, + "loss": 3.789005234011711e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1283, + "tokens/total": 19371600, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 414767 + }, + { + "epoch": 20.0625, + "grad_norm": 8.751199493417516e-05, + "learning_rate": 4.017173234367164e-05, + "loss": 1.0197391020483337e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1284, + "tokens/total": 19386784, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 415088 + }, + { + "epoch": 20.078125, + "grad_norm": 1.2344601600489113e-05, + "learning_rate": 4.0103161764203546e-05, + "loss": 2.61034529103199e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1285, + "tokens/total": 19401948, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 415416 + }, + { + "epoch": 20.09375, + "grad_norm": 3.893056054948829e-05, + "learning_rate": 4.0034630009432986e-05, + "loss": 6.713938773827977e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1286, + "tokens/total": 19417056, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 415738 + }, + { + "epoch": 20.109375, + "grad_norm": 9.7437859949423e-06, + "learning_rate": 3.996613725796998e-05, + "loss": 2.2500825025417726e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1287, + "tokens/total": 19432244, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 416064 + }, + { + "epoch": 20.125, + "grad_norm": 2.3369653717963956e-05, + "learning_rate": 3.989768368832291e-05, + "loss": 6.206687430676538e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1288, + "tokens/total": 19447376, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 416388 + }, + { + "epoch": 20.140625, + "grad_norm": 2.4898759875213727e-05, + "learning_rate": 3.9829269478898026e-05, + "loss": 4.2766271235450404e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1289, + "tokens/total": 19462444, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 416712 + }, + { + "epoch": 20.15625, + "grad_norm": 0.00010899813787546009, + "learning_rate": 3.9760894807999004e-05, + "loss": 1.8242762962472625e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1290, + "tokens/total": 19477532, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 417038 + }, + { + "epoch": 20.171875, + "grad_norm": 3.087766162934713e-05, + "learning_rate": 3.969255985382646e-05, + "loss": 6.70617396281159e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1291, + "tokens/total": 19492660, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 417359 + }, + { + "epoch": 20.1875, + "grad_norm": 2.9831460778950714e-05, + "learning_rate": 3.9624264794477515e-05, + "loss": 6.030361419107066e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1292, + "tokens/total": 19507680, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 417679 + }, + { + "epoch": 20.203125, + "grad_norm": 2.1109244698891416e-05, + "learning_rate": 3.9556009807945335e-05, + "loss": 4.919511411571875e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1293, + "tokens/total": 19522760, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 418003 + }, + { + "epoch": 20.21875, + "grad_norm": 1.726173832139466e-05, + "learning_rate": 3.948779507211859e-05, + "loss": 3.0583007060158707e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1294, + "tokens/total": 19537868, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 418329 + }, + { + "epoch": 20.234375, + "grad_norm": 2.430771928629838e-05, + "learning_rate": 3.9419620764781094e-05, + "loss": 4.1206669720850186e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1295, + "tokens/total": 19552928, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 418652 + }, + { + "epoch": 20.25, + "grad_norm": 4.2676791053963825e-05, + "learning_rate": 3.935148706361128e-05, + "loss": 1.0665678473742446e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1296, + "tokens/total": 19568036, + "tokens/train_per_sec_per_gpu": 9.0, + "tokens/trainable": 418972 + }, + { + "epoch": 20.265625, + "grad_norm": 2.4619348550913855e-05, + "learning_rate": 3.928339414618174e-05, + "loss": 4.1428040731261717e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1297, + "tokens/total": 19583140, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 419298 + }, + { + "epoch": 20.28125, + "grad_norm": 4.093311872566119e-05, + "learning_rate": 3.921534218995878e-05, + "loss": 7.646973472219543e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1298, + "tokens/total": 19598276, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 419618 + }, + { + "epoch": 20.296875, + "grad_norm": 2.8151902370154858e-05, + "learning_rate": 3.914733137230198e-05, + "loss": 6.980527018640714e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1299, + "tokens/total": 19613396, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 419939 + }, + { + "epoch": 20.3125, + "grad_norm": 1.7838030544226058e-05, + "learning_rate": 3.907936187046366e-05, + "loss": 4.2604273176038987e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1300, + "tokens/total": 19628456, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 420263 + }, + { + "epoch": 20.328125, + "grad_norm": 1.602999327587895e-05, + "learning_rate": 3.9011433861588484e-05, + "loss": 3.6491809396466124e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1301, + "tokens/total": 19643552, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 420585 + }, + { + "epoch": 20.34375, + "grad_norm": 2.7544232580112293e-05, + "learning_rate": 3.8943547522712976e-05, + "loss": 6.235106297935999e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1302, + "tokens/total": 19658608, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 420910 + }, + { + "epoch": 20.359375, + "grad_norm": 1.6511277863173746e-05, + "learning_rate": 3.8875703030765056e-05, + "loss": 3.941243562621821e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1303, + "tokens/total": 19673628, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 421233 + }, + { + "epoch": 20.375, + "grad_norm": 4.764023469761014e-05, + "learning_rate": 3.880790056256357e-05, + "loss": 8.486248361805337e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1304, + "tokens/total": 19688680, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 421558 + }, + { + "epoch": 20.390625, + "grad_norm": 2.7197236704523675e-05, + "learning_rate": 3.874014029481785e-05, + "loss": 5.120452328810643e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1305, + "tokens/total": 19703752, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 421887 + }, + { + "epoch": 20.40625, + "grad_norm": 2.986555773532018e-05, + "learning_rate": 3.867242240412725e-05, + "loss": 4.017883838969283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1306, + "tokens/total": 19718836, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 422209 + }, + { + "epoch": 20.421875, + "grad_norm": 1.1848346730403136e-05, + "learning_rate": 3.860474706698067e-05, + "loss": 2.5680850512799225e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1307, + "tokens/total": 19733936, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 422535 + }, + { + "epoch": 20.4375, + "grad_norm": 1.574533962411806e-05, + "learning_rate": 3.8537114459756106e-05, + "loss": 2.948406745417742e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1308, + "tokens/total": 19749040, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 422864 + }, + { + "epoch": 20.453125, + "grad_norm": 1.0846362783922814e-05, + "learning_rate": 3.84695247587202e-05, + "loss": 2.2772478303068056e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1309, + "tokens/total": 19764032, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 423186 + }, + { + "epoch": 20.46875, + "grad_norm": 0.0033920209389179945, + "learning_rate": 3.840197814002774e-05, + "loss": 2.1289038159011398e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1310, + "tokens/total": 19779116, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 423505 + }, + { + "epoch": 20.484375, + "grad_norm": 8.457337389700115e-05, + "learning_rate": 3.8334474779721264e-05, + "loss": 1.1761197811210877e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1311, + "tokens/total": 19794200, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 423828 + }, + { + "epoch": 20.5, + "grad_norm": 1.905378303490579e-05, + "learning_rate": 3.826701485373056e-05, + "loss": 4.6443790324701695e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1312, + "tokens/total": 19809224, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 424147 + }, + { + "epoch": 20.515625, + "grad_norm": 1.9468561731628142e-05, + "learning_rate": 3.819959853787222e-05, + "loss": 4.0046438698482234e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1313, + "tokens/total": 19824340, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 424469 + }, + { + "epoch": 20.53125, + "grad_norm": 1.520846944913501e-05, + "learning_rate": 3.813222600784915e-05, + "loss": 3.666233396870666e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1314, + "tokens/total": 19839468, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 424792 + }, + { + "epoch": 20.546875, + "grad_norm": 1.2422530744515825e-05, + "learning_rate": 3.806489743925016e-05, + "loss": 2.69318150003528e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1315, + "tokens/total": 19854660, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 425113 + }, + { + "epoch": 20.5625, + "grad_norm": 3.5566321457736194e-05, + "learning_rate": 3.799761300754953e-05, + "loss": 7.417354481731309e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1316, + "tokens/total": 19869740, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 425433 + }, + { + "epoch": 20.578125, + "grad_norm": 1.982124194910284e-05, + "learning_rate": 3.7930372888106404e-05, + "loss": 4.496529300013208e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1317, + "tokens/total": 19884888, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 425758 + }, + { + "epoch": 20.59375, + "grad_norm": 1.7249845768674277e-05, + "learning_rate": 3.7863177256164526e-05, + "loss": 3.7920335671515204e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1318, + "tokens/total": 19900020, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 426084 + }, + { + "epoch": 20.609375, + "grad_norm": 5.1534356316551566e-05, + "learning_rate": 3.779602628685167e-05, + "loss": 9.256779662791814e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1319, + "tokens/total": 19915184, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 426408 + }, + { + "epoch": 20.625, + "grad_norm": 2.613913602544926e-05, + "learning_rate": 3.772892015517924e-05, + "loss": 4.896898531114857e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1320, + "tokens/total": 19930336, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 426731 + }, + { + "epoch": 20.640625, + "grad_norm": 4.6634417230961844e-05, + "learning_rate": 3.7661859036041666e-05, + "loss": 8.671658520142955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1321, + "tokens/total": 19945444, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 427049 + }, + { + "epoch": 20.65625, + "grad_norm": 1.5258494386216626e-05, + "learning_rate": 3.7594843104216215e-05, + "loss": 3.569441275885765e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1322, + "tokens/total": 19960440, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 427374 + }, + { + "epoch": 20.671875, + "grad_norm": 3.812923387158662e-05, + "learning_rate": 3.752787253436231e-05, + "loss": 5.320707145983761e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1323, + "tokens/total": 19975548, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 427699 + }, + { + "epoch": 20.6875, + "grad_norm": 2.0295579815865494e-05, + "learning_rate": 3.746094750102113e-05, + "loss": 4.953954544362205e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1324, + "tokens/total": 19990704, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 428020 + }, + { + "epoch": 20.703125, + "grad_norm": 1.9560695363907143e-05, + "learning_rate": 3.739406817861523e-05, + "loss": 4.5531837145063037e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1325, + "tokens/total": 20005780, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 428341 + }, + { + "epoch": 20.71875, + "grad_norm": 1.2916993000544608e-05, + "learning_rate": 3.732723474144797e-05, + "loss": 2.939507623977988e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1326, + "tokens/total": 20020948, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 428661 + }, + { + "epoch": 20.734375, + "grad_norm": 2.1383062630775385e-05, + "learning_rate": 3.726044736370319e-05, + "loss": 3.3149549949484935e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1327, + "tokens/total": 20035924, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 428982 + }, + { + "epoch": 20.75, + "grad_norm": 1.8255685063195415e-05, + "learning_rate": 3.7193706219444647e-05, + "loss": 4.2838092895181035e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1328, + "tokens/total": 20051084, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 429312 + }, + { + "epoch": 20.765625, + "grad_norm": 2.6614483431330882e-05, + "learning_rate": 3.712701148261559e-05, + "loss": 6.665676437478396e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1329, + "tokens/total": 20066216, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 429641 + }, + { + "epoch": 20.78125, + "grad_norm": 3.3013857319019735e-05, + "learning_rate": 3.706036332703836e-05, + "loss": 7.645631967534428e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1330, + "tokens/total": 20081184, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 429963 + }, + { + "epoch": 20.796875, + "grad_norm": 2.755112109298352e-05, + "learning_rate": 3.6993761926413885e-05, + "loss": 5.03282706176833e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1331, + "tokens/total": 20096188, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 430286 + }, + { + "epoch": 20.8125, + "grad_norm": 5.9018078900408e-05, + "learning_rate": 3.692720745432119e-05, + "loss": 5.538614118449914e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1332, + "tokens/total": 20111212, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 430609 + }, + { + "epoch": 20.828125, + "grad_norm": 1.855276059359312e-05, + "learning_rate": 3.686070008421706e-05, + "loss": 3.8810483715678856e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1333, + "tokens/total": 20126416, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 430933 + }, + { + "epoch": 20.84375, + "grad_norm": 2.2627154976362363e-05, + "learning_rate": 3.6794239989435483e-05, + "loss": 4.664634047912841e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1334, + "tokens/total": 20141480, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 431258 + }, + { + "epoch": 20.859375, + "grad_norm": 3.210284194210544e-05, + "learning_rate": 3.672782734318728e-05, + "loss": 4.6296821665237076e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1335, + "tokens/total": 20156568, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 431582 + }, + { + "epoch": 20.875, + "grad_norm": 1.1180003639310598e-05, + "learning_rate": 3.666146231855953e-05, + "loss": 2.397990783720161e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1336, + "tokens/total": 20171636, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 431909 + }, + { + "epoch": 20.890625, + "grad_norm": 3.218363781343214e-05, + "learning_rate": 3.659514508851525e-05, + "loss": 4.010639429452567e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1337, + "tokens/total": 20186768, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 432232 + }, + { + "epoch": 20.90625, + "grad_norm": 1.935248110385146e-05, + "learning_rate": 3.6528875825892896e-05, + "loss": 4.1888344526341825e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1338, + "tokens/total": 20201984, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 432557 + }, + { + "epoch": 20.921875, + "grad_norm": 4.1006423998624086e-05, + "learning_rate": 3.646265470340593e-05, + "loss": 6.810020636294212e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1339, + "tokens/total": 20217052, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 432880 + }, + { + "epoch": 20.9375, + "grad_norm": 4.1294082620879635e-05, + "learning_rate": 3.6396481893642274e-05, + "loss": 3.5704977108252933e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1340, + "tokens/total": 20232188, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 433199 + }, + { + "epoch": 20.953125, + "grad_norm": 9.705930096970405e-06, + "learning_rate": 3.633035756906401e-05, + "loss": 2.230856779306123e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1341, + "tokens/total": 20247212, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 433526 + }, + { + "epoch": 20.96875, + "grad_norm": 3.8285030314000323e-05, + "learning_rate": 3.626428190200683e-05, + "loss": 8.604350796304061e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1342, + "tokens/total": 20262240, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 433848 + }, + { + "epoch": 20.984375, + "grad_norm": 4.295492544770241e-05, + "learning_rate": 3.619825506467961e-05, + "loss": 7.112867024261504e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1343, + "tokens/total": 20277504, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 434171 + }, + { + "epoch": 21.0, + "grad_norm": 8.089939365163445e-05, + "learning_rate": 3.6132277229163934e-05, + "loss": 4.359737317827239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1344, + "tokens/total": 20292592, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 434490 + }, + { + "epoch": 21.015625, + "grad_norm": 2.02058090508217e-05, + "learning_rate": 3.6066348567413746e-05, + "loss": 3.6542419934448844e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1345, + "tokens/total": 20307648, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 434816 + }, + { + "epoch": 21.03125, + "grad_norm": 2.417656833131332e-05, + "learning_rate": 3.6000469251254764e-05, + "loss": 5.967320362287865e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1346, + "tokens/total": 20322720, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 435137 + }, + { + "epoch": 21.046875, + "grad_norm": 3.0070756110944785e-05, + "learning_rate": 3.5934639452384176e-05, + "loss": 6.289857310548541e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1347, + "tokens/total": 20337848, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 435463 + }, + { + "epoch": 21.0625, + "grad_norm": 2.0722145563922822e-05, + "learning_rate": 3.586885934237001e-05, + "loss": 3.516259141633782e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1348, + "tokens/total": 20353052, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 435779 + }, + { + "epoch": 21.078125, + "grad_norm": 4.012781573692337e-05, + "learning_rate": 3.580312909265088e-05, + "loss": 9.19224873996427e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1349, + "tokens/total": 20368052, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 436101 + }, + { + "epoch": 21.09375, + "grad_norm": 2.3627701011719182e-05, + "learning_rate": 3.573744887453543e-05, + "loss": 4.6980824208731065e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1350, + "tokens/total": 20383192, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 436423 + }, + { + "epoch": 21.109375, + "grad_norm": 2.317241523996927e-05, + "learning_rate": 3.5671818859201886e-05, + "loss": 4.0396000144937716e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1351, + "tokens/total": 20398284, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 436746 + }, + { + "epoch": 21.125, + "grad_norm": 0.00020711467368528247, + "learning_rate": 3.5606239217697643e-05, + "loss": 3.3532040788486484e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1352, + "tokens/total": 20413336, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 437072 + }, + { + "epoch": 21.140625, + "grad_norm": 2.7252048312220722e-05, + "learning_rate": 3.554071012093886e-05, + "loss": 3.3029988344424055e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1353, + "tokens/total": 20428516, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 437394 + }, + { + "epoch": 21.15625, + "grad_norm": 1.8788852685247548e-05, + "learning_rate": 3.547523173970989e-05, + "loss": 3.8291631199172116e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1354, + "tokens/total": 20443676, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 437717 + }, + { + "epoch": 21.171875, + "grad_norm": 2.638211299199611e-05, + "learning_rate": 3.5409804244662945e-05, + "loss": 4.001089450866857e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1355, + "tokens/total": 20458780, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 438046 + }, + { + "epoch": 21.1875, + "grad_norm": 0.0002130104403477162, + "learning_rate": 3.5344427806317626e-05, + "loss": 1.9299568521091715e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1356, + "tokens/total": 20473920, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 438374 + }, + { + "epoch": 21.203125, + "grad_norm": 1.3580175618699286e-05, + "learning_rate": 3.527910259506041e-05, + "loss": 3.07827491496937e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1357, + "tokens/total": 20489016, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 438699 + }, + { + "epoch": 21.21875, + "grad_norm": 1.382375558023341e-05, + "learning_rate": 3.521382878114438e-05, + "loss": 2.5059048880393675e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1358, + "tokens/total": 20504116, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 439026 + }, + { + "epoch": 21.234375, + "grad_norm": 2.0561061319313012e-05, + "learning_rate": 3.514860653468853e-05, + "loss": 4.2059338056787965e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1359, + "tokens/total": 20519248, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 439354 + }, + { + "epoch": 21.25, + "grad_norm": 3.2718318834668025e-05, + "learning_rate": 3.508343602567756e-05, + "loss": 5.096511017654848e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1360, + "tokens/total": 20534400, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 439677 + }, + { + "epoch": 21.265625, + "grad_norm": 1.8132355762645602e-05, + "learning_rate": 3.501831742396126e-05, + "loss": 4.660159049763024e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1361, + "tokens/total": 20549576, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 440001 + }, + { + "epoch": 21.28125, + "grad_norm": 1.9878261809935793e-05, + "learning_rate": 3.495325089925419e-05, + "loss": 4.346134687693848e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1362, + "tokens/total": 20564652, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 440326 + }, + { + "epoch": 21.296875, + "grad_norm": 2.4218694306910038e-05, + "learning_rate": 3.488823662113516e-05, + "loss": 6.090989472795627e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1363, + "tokens/total": 20579792, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 440651 + }, + { + "epoch": 21.3125, + "grad_norm": 4.3350763007765636e-05, + "learning_rate": 3.482327475904682e-05, + "loss": 3.93418105204546e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1364, + "tokens/total": 20594872, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 440977 + }, + { + "epoch": 21.328125, + "grad_norm": 1.5915416952339e-05, + "learning_rate": 3.475836548229518e-05, + "loss": 3.829985359971033e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1365, + "tokens/total": 20610000, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 441300 + }, + { + "epoch": 21.34375, + "grad_norm": 1.5299518054234795e-05, + "learning_rate": 3.469350896004928e-05, + "loss": 3.5610281656772713e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1366, + "tokens/total": 20625080, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 441620 + }, + { + "epoch": 21.359375, + "grad_norm": 1.2501961464295164e-05, + "learning_rate": 3.4628705361340545e-05, + "loss": 2.7892880893887195e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1367, + "tokens/total": 20640196, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 441941 + }, + { + "epoch": 21.375, + "grad_norm": 3.561500125215389e-05, + "learning_rate": 3.45639548550626e-05, + "loss": 4.332065373091609e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1368, + "tokens/total": 20655112, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 442264 + }, + { + "epoch": 21.390625, + "grad_norm": 1.2590972801262978e-05, + "learning_rate": 3.449925760997062e-05, + "loss": 3.052118984214758e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1369, + "tokens/total": 20670272, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 442584 + }, + { + "epoch": 21.40625, + "grad_norm": 4.511540464591235e-05, + "learning_rate": 3.443461379468098e-05, + "loss": 7.487174116249662e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1370, + "tokens/total": 20685424, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 442904 + }, + { + "epoch": 21.421875, + "grad_norm": 1.5140273717406671e-05, + "learning_rate": 3.437002357767079e-05, + "loss": 3.500715308746294e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1371, + "tokens/total": 20700392, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 443226 + }, + { + "epoch": 21.4375, + "grad_norm": 1.9094466551905498e-05, + "learning_rate": 3.430548712727754e-05, + "loss": 4.274221510058851e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1372, + "tokens/total": 20715468, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 443549 + }, + { + "epoch": 21.453125, + "grad_norm": 0.00011022774560842663, + "learning_rate": 3.424100461169853e-05, + "loss": 3.993381199052237e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1373, + "tokens/total": 20730692, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 443871 + }, + { + "epoch": 21.46875, + "grad_norm": 3.622242729761638e-05, + "learning_rate": 3.4176576198990484e-05, + "loss": 6.719874363625422e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1374, + "tokens/total": 20745688, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 444194 + }, + { + "epoch": 21.484375, + "grad_norm": 0.00015885967877693474, + "learning_rate": 3.4112202057069164e-05, + "loss": 1.5300304312404478e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1375, + "tokens/total": 20760792, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 444514 + }, + { + "epoch": 21.5, + "grad_norm": 1.945311487361323e-05, + "learning_rate": 3.404788235370884e-05, + "loss": 3.433374899941555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1376, + "tokens/total": 20776004, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 444839 + }, + { + "epoch": 21.515625, + "grad_norm": 2.8483349524321966e-05, + "learning_rate": 3.398361725654198e-05, + "loss": 6.100366931605095e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1377, + "tokens/total": 20791076, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 445164 + }, + { + "epoch": 21.53125, + "grad_norm": 3.488637230475433e-05, + "learning_rate": 3.3919406933058646e-05, + "loss": 8.186943887267262e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1378, + "tokens/total": 20806216, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 445488 + }, + { + "epoch": 21.546875, + "grad_norm": 2.3319902538787574e-05, + "learning_rate": 3.3855251550606205e-05, + "loss": 4.3252754267086857e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1379, + "tokens/total": 20821228, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 445810 + }, + { + "epoch": 21.5625, + "grad_norm": 2.5937148166121915e-05, + "learning_rate": 3.379115127638879e-05, + "loss": 5.454592155729188e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1380, + "tokens/total": 20836312, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 446130 + }, + { + "epoch": 21.578125, + "grad_norm": 2.1996815121383406e-05, + "learning_rate": 3.3727106277466974e-05, + "loss": 5.092089168101666e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1381, + "tokens/total": 20851404, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 446458 + }, + { + "epoch": 21.59375, + "grad_norm": 3.166968235746026e-05, + "learning_rate": 3.366311672075722e-05, + "loss": 4.988384603166196e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1382, + "tokens/total": 20866552, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 446782 + }, + { + "epoch": 21.609375, + "grad_norm": 1.8885031749960035e-05, + "learning_rate": 3.359918277303149e-05, + "loss": 4.241364308654738e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1383, + "tokens/total": 20881692, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 447106 + }, + { + "epoch": 21.625, + "grad_norm": 1.8223739971290343e-05, + "learning_rate": 3.353530460091683e-05, + "loss": 3.6601818464987446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1384, + "tokens/total": 20896784, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 447429 + }, + { + "epoch": 21.640625, + "grad_norm": 1.5592120689689182e-05, + "learning_rate": 3.3471482370894955e-05, + "loss": 3.599904516704555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1385, + "tokens/total": 20911884, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 447751 + }, + { + "epoch": 21.65625, + "grad_norm": 2.61320765275741e-05, + "learning_rate": 3.340771624930173e-05, + "loss": 6.798213121328445e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1386, + "tokens/total": 20927016, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 448076 + }, + { + "epoch": 21.671875, + "grad_norm": 1.2948389667144511e-05, + "learning_rate": 3.3344006402326814e-05, + "loss": 2.8321449008217314e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1387, + "tokens/total": 20942124, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 448400 + }, + { + "epoch": 21.6875, + "grad_norm": 7.439892215188593e-05, + "learning_rate": 3.3280352996013184e-05, + "loss": 1.1646508255580557e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1388, + "tokens/total": 20957224, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 448724 + }, + { + "epoch": 21.703125, + "grad_norm": 2.206319732067641e-05, + "learning_rate": 3.3216756196256724e-05, + "loss": 5.354059453566151e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1389, + "tokens/total": 20972224, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 449047 + }, + { + "epoch": 21.71875, + "grad_norm": 1.7033060430549085e-05, + "learning_rate": 3.315321616880582e-05, + "loss": 4.097151418136491e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1390, + "tokens/total": 20987256, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 449366 + }, + { + "epoch": 21.734375, + "grad_norm": 5.980149217066355e-05, + "learning_rate": 3.308973307926086e-05, + "loss": 1.0788191957544768e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1391, + "tokens/total": 21002392, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 449693 + }, + { + "epoch": 21.75, + "grad_norm": 1.279412299481919e-05, + "learning_rate": 3.302630709307385e-05, + "loss": 2.916557093612937e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1392, + "tokens/total": 21017628, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 450013 + }, + { + "epoch": 21.765625, + "grad_norm": 1.3749671779805794e-05, + "learning_rate": 3.2962938375547954e-05, + "loss": 3.0294097541627707e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1393, + "tokens/total": 21032684, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 450333 + }, + { + "epoch": 21.78125, + "grad_norm": 3.925274722860195e-05, + "learning_rate": 3.2899627091837135e-05, + "loss": 8.47141905069293e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1394, + "tokens/total": 21047720, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 450655 + }, + { + "epoch": 21.796875, + "grad_norm": 2.545003553677816e-05, + "learning_rate": 3.283637340694558e-05, + "loss": 4.01996572918506e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1395, + "tokens/total": 21062752, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 450973 + }, + { + "epoch": 21.8125, + "grad_norm": 3.675095649668947e-05, + "learning_rate": 3.277317748572745e-05, + "loss": 6.526311153720599e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1396, + "tokens/total": 21077856, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 451303 + }, + { + "epoch": 21.828125, + "grad_norm": 0.00010848904639715329, + "learning_rate": 3.271003949288628e-05, + "loss": 1.7340219073957996e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1397, + "tokens/total": 21092976, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 451623 + }, + { + "epoch": 21.84375, + "grad_norm": 2.7267389668850228e-05, + "learning_rate": 3.264695959297474e-05, + "loss": 5.250045660432079e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1398, + "tokens/total": 21108192, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 451953 + }, + { + "epoch": 21.859375, + "grad_norm": 2.812142156471964e-05, + "learning_rate": 3.258393795039395e-05, + "loss": 4.434831453181687e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1399, + "tokens/total": 21123320, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 452278 + }, + { + "epoch": 21.875, + "grad_norm": 1.2944638001499698e-05, + "learning_rate": 3.252097472939332e-05, + "loss": 2.057654029385958e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1400, + "tokens/total": 21138400, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 452601 + }, + { + "epoch": 21.890625, + "grad_norm": 7.94857169239549e-06, + "learning_rate": 3.2458070094069946e-05, + "loss": 1.5012436449524102e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1401, + "tokens/total": 21153508, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 452926 + }, + { + "epoch": 21.90625, + "grad_norm": 1.532313399366103e-05, + "learning_rate": 3.2395224208368245e-05, + "loss": 3.483124260128534e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1402, + "tokens/total": 21168704, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 453247 + }, + { + "epoch": 21.921875, + "grad_norm": 1.4071501027501654e-05, + "learning_rate": 3.233243723607949e-05, + "loss": 3.355221167566924e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1403, + "tokens/total": 21183680, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 453565 + }, + { + "epoch": 21.9375, + "grad_norm": 1.2261375559319276e-05, + "learning_rate": 3.226970934084148e-05, + "loss": 2.668107583758683e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1404, + "tokens/total": 21198864, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 453891 + }, + { + "epoch": 21.953125, + "grad_norm": 9.967711775971111e-06, + "learning_rate": 3.220704068613799e-05, + "loss": 2.1158732010917447e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1405, + "tokens/total": 21213996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 454212 + }, + { + "epoch": 21.96875, + "grad_norm": 2.206218232458923e-05, + "learning_rate": 3.2144431435298405e-05, + "loss": 4.616390469891485e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1406, + "tokens/total": 21229080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 454535 + }, + { + "epoch": 21.984375, + "grad_norm": 3.5783512430498376e-05, + "learning_rate": 3.20818817514973e-05, + "loss": 7.751223733976076e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1407, + "tokens/total": 21244184, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 454859 + }, + { + "epoch": 22.0, + "grad_norm": 2.5069020921364427e-05, + "learning_rate": 3.201939179775399e-05, + "loss": 4.1226422808904317e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1408, + "tokens/total": 21259228, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 455180 + }, + { + "epoch": 22.015625, + "grad_norm": 7.025229569990188e-05, + "learning_rate": 3.195696173693217e-05, + "loss": 9.291585456594476e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1409, + "tokens/total": 21274260, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 455504 + }, + { + "epoch": 22.03125, + "grad_norm": 1.2059239452355541e-05, + "learning_rate": 3.1894591731739376e-05, + "loss": 2.611920990602812e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1410, + "tokens/total": 21289316, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 455830 + }, + { + "epoch": 22.046875, + "grad_norm": 2.1008276235079393e-05, + "learning_rate": 3.183228194472665e-05, + "loss": 3.9205099255923415e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1411, + "tokens/total": 21304508, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 456159 + }, + { + "epoch": 22.0625, + "grad_norm": 1.542438803880941e-05, + "learning_rate": 3.177003253828808e-05, + "loss": 3.3816837685662904e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1412, + "tokens/total": 21319516, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 456481 + }, + { + "epoch": 22.078125, + "grad_norm": 2.124123420799151e-05, + "learning_rate": 3.1707843674660454e-05, + "loss": 4.588560784668516e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1413, + "tokens/total": 21334676, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 456807 + }, + { + "epoch": 22.09375, + "grad_norm": 2.973677692352794e-05, + "learning_rate": 3.164571551592265e-05, + "loss": 6.27240353878733e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1414, + "tokens/total": 21349732, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 457126 + }, + { + "epoch": 22.109375, + "grad_norm": 1.26044242279022e-05, + "learning_rate": 3.1583648223995435e-05, + "loss": 2.4734006842663803e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1415, + "tokens/total": 21364788, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 457448 + }, + { + "epoch": 22.125, + "grad_norm": 2.3232028979691677e-05, + "learning_rate": 3.152164196064089e-05, + "loss": 5.456313374452293e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1416, + "tokens/total": 21379876, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 457773 + }, + { + "epoch": 22.140625, + "grad_norm": 1.5875753888394684e-05, + "learning_rate": 3.1459696887462096e-05, + "loss": 2.020842515548793e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1417, + "tokens/total": 21395092, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 458101 + }, + { + "epoch": 22.15625, + "grad_norm": 3.983770511695184e-05, + "learning_rate": 3.139781316590258e-05, + "loss": 5.034479499954614e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1418, + "tokens/total": 21410224, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 458425 + }, + { + "epoch": 22.171875, + "grad_norm": 3.54419244104065e-05, + "learning_rate": 3.1335990957246054e-05, + "loss": 6.038260380591964e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1419, + "tokens/total": 21425260, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 458749 + }, + { + "epoch": 22.1875, + "grad_norm": 1.5188197721727192e-05, + "learning_rate": 3.127423042261582e-05, + "loss": 3.363896894370555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1420, + "tokens/total": 21440388, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 459074 + }, + { + "epoch": 22.203125, + "grad_norm": 1.1933226232940797e-05, + "learning_rate": 3.121253172297457e-05, + "loss": 2.1585273657365178e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1421, + "tokens/total": 21455424, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 459396 + }, + { + "epoch": 22.21875, + "grad_norm": 2.843527363438625e-05, + "learning_rate": 3.115089501912369e-05, + "loss": 4.771538897330174e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1422, + "tokens/total": 21470464, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 459711 + }, + { + "epoch": 22.234375, + "grad_norm": 9.712856990518048e-05, + "learning_rate": 3.108932047170311e-05, + "loss": 1.4452431287281797e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1423, + "tokens/total": 21485624, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 460032 + }, + { + "epoch": 22.25, + "grad_norm": 3.878276402247138e-05, + "learning_rate": 3.102780824119071e-05, + "loss": 8.415014463025727e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1424, + "tokens/total": 21500744, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 460354 + }, + { + "epoch": 22.265625, + "grad_norm": 2.7282203518552706e-05, + "learning_rate": 3.096635848790197e-05, + "loss": 4.4517054220705177e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1425, + "tokens/total": 21515860, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 460677 + }, + { + "epoch": 22.28125, + "grad_norm": 1.2476179108489305e-05, + "learning_rate": 3.090497137198953e-05, + "loss": 2.1222012946964242e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1426, + "tokens/total": 21531128, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 461002 + }, + { + "epoch": 22.296875, + "grad_norm": 1.4837150956736878e-05, + "learning_rate": 3.0843647053442784e-05, + "loss": 3.349716166667349e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1427, + "tokens/total": 21546184, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 461326 + }, + { + "epoch": 22.3125, + "grad_norm": 2.972052425320726e-05, + "learning_rate": 3.078238569208749e-05, + "loss": 7.160158475016942e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1428, + "tokens/total": 21561224, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 461648 + }, + { + "epoch": 22.328125, + "grad_norm": 6.221830699359998e-05, + "learning_rate": 3.072118744758528e-05, + "loss": 6.58552608001628e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1429, + "tokens/total": 21576276, + "tokens/train_per_sec_per_gpu": 8.69, + "tokens/trainable": 461969 + }, + { + "epoch": 22.34375, + "grad_norm": 1.93275191122666e-05, + "learning_rate": 3.066005247943333e-05, + "loss": 5.166448886484432e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1430, + "tokens/total": 21591356, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 462296 + }, + { + "epoch": 22.359375, + "grad_norm": 1.22199380712118e-05, + "learning_rate": 3.059898094696385e-05, + "loss": 2.58158110000295e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1431, + "tokens/total": 21606484, + "tokens/train_per_sec_per_gpu": 10.2, + "tokens/trainable": 462627 + }, + { + "epoch": 22.375, + "grad_norm": 1.7747808669810183e-05, + "learning_rate": 3.053797300934379e-05, + "loss": 3.753342525669723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1432, + "tokens/total": 21621664, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 462954 + }, + { + "epoch": 22.390625, + "grad_norm": 9.546996807330288e-06, + "learning_rate": 3.0477028825574316e-05, + "loss": 2.0428463187727175e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1433, + "tokens/total": 21636692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 463276 + }, + { + "epoch": 22.40625, + "grad_norm": 3.5236796975368634e-05, + "learning_rate": 3.041614855449042e-05, + "loss": 4.3144859773747157e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1434, + "tokens/total": 21651872, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 463600 + }, + { + "epoch": 22.421875, + "grad_norm": 1.5360086763394065e-05, + "learning_rate": 3.035533235476055e-05, + "loss": 3.5833028277920675e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1435, + "tokens/total": 21666928, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 463925 + }, + { + "epoch": 22.4375, + "grad_norm": 2.7332183890393935e-05, + "learning_rate": 3.029458038488618e-05, + "loss": 5.785714733974601e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1436, + "tokens/total": 21682028, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 464252 + }, + { + "epoch": 22.453125, + "grad_norm": 1.4617397027905099e-05, + "learning_rate": 3.0233892803201357e-05, + "loss": 3.459413164819125e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1437, + "tokens/total": 21697100, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 464575 + }, + { + "epoch": 22.46875, + "grad_norm": 4.329115108703263e-05, + "learning_rate": 3.017326976787233e-05, + "loss": 1.0418859801575309e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1438, + "tokens/total": 21712152, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 464899 + }, + { + "epoch": 22.484375, + "grad_norm": 1.8945724150398746e-05, + "learning_rate": 3.011271143689711e-05, + "loss": 4.1202162037734524e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1439, + "tokens/total": 21727268, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 465215 + }, + { + "epoch": 22.5, + "grad_norm": 1.3458920875564218e-05, + "learning_rate": 3.005221796810513e-05, + "loss": 2.976412361022085e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1440, + "tokens/total": 21742364, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 465535 + }, + { + "epoch": 22.515625, + "grad_norm": 2.666440195753239e-05, + "learning_rate": 2.999178951915666e-05, + "loss": 4.149945311837655e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1441, + "tokens/total": 21757360, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 465859 + }, + { + "epoch": 22.53125, + "grad_norm": 3.612696673371829e-05, + "learning_rate": 2.993142624754265e-05, + "loss": 5.721192337659886e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1442, + "tokens/total": 21772412, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 466183 + }, + { + "epoch": 22.546875, + "grad_norm": 2.5722118152771145e-05, + "learning_rate": 2.9871128310584094e-05, + "loss": 6.259962788135454e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1443, + "tokens/total": 21787420, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 466504 + }, + { + "epoch": 22.5625, + "grad_norm": 1.545891245768871e-05, + "learning_rate": 2.9810895865431732e-05, + "loss": 2.4706272938601614e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1444, + "tokens/total": 21802536, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 466828 + }, + { + "epoch": 22.578125, + "grad_norm": 4.053384327562526e-05, + "learning_rate": 2.975072906906563e-05, + "loss": 6.286929306043021e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1445, + "tokens/total": 21817480, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 467147 + }, + { + "epoch": 22.59375, + "grad_norm": 1.6542693629162386e-05, + "learning_rate": 2.9690628078294714e-05, + "loss": 3.3166907087434083e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1446, + "tokens/total": 21832616, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 467470 + }, + { + "epoch": 22.609375, + "grad_norm": 2.880009196815081e-05, + "learning_rate": 2.9630593049756472e-05, + "loss": 3.8895046827747137e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1447, + "tokens/total": 21847652, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 467793 + }, + { + "epoch": 22.625, + "grad_norm": 0.00012743678234983236, + "learning_rate": 2.957062413991643e-05, + "loss": 1.2314771993260365e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1448, + "tokens/total": 21862660, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 468113 + }, + { + "epoch": 22.640625, + "grad_norm": 8.75893692864338e-06, + "learning_rate": 2.9510721505067806e-05, + "loss": 1.9332961187501496e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1449, + "tokens/total": 21877712, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 468434 + }, + { + "epoch": 22.65625, + "grad_norm": 1.213582800119184e-05, + "learning_rate": 2.9450885301331065e-05, + "loss": 2.5199062747560674e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1450, + "tokens/total": 21892844, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 468761 + }, + { + "epoch": 22.671875, + "grad_norm": 1.3681916243513115e-05, + "learning_rate": 2.9391115684653612e-05, + "loss": 3.059140567529539e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1451, + "tokens/total": 21908044, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 469089 + }, + { + "epoch": 22.6875, + "grad_norm": 0.00021077391284052283, + "learning_rate": 2.9331412810809232e-05, + "loss": 2.1559358174272347e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1452, + "tokens/total": 21923252, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 469410 + }, + { + "epoch": 22.703125, + "grad_norm": 1.850956687121652e-05, + "learning_rate": 2.9271776835397785e-05, + "loss": 4.104427944184863e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1453, + "tokens/total": 21938288, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 469737 + }, + { + "epoch": 22.71875, + "grad_norm": 1.5558971426798962e-05, + "learning_rate": 2.9212207913844775e-05, + "loss": 3.324629744838603e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1454, + "tokens/total": 21953392, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 470060 + }, + { + "epoch": 22.734375, + "grad_norm": 1.2645864444493782e-05, + "learning_rate": 2.915270620140099e-05, + "loss": 2.963989800264244e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1455, + "tokens/total": 21968500, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 470381 + }, + { + "epoch": 22.75, + "grad_norm": 1.6443578715552576e-05, + "learning_rate": 2.909327185314199e-05, + "loss": 2.749569887328107e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1456, + "tokens/total": 21983616, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 470706 + }, + { + "epoch": 22.765625, + "grad_norm": 2.2733473088010214e-05, + "learning_rate": 2.903390502396781e-05, + "loss": 2.926252022916742e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1457, + "tokens/total": 21998636, + "tokens/train_per_sec_per_gpu": 8.62, + "tokens/trainable": 471026 + }, + { + "epoch": 22.78125, + "grad_norm": 8.136017640936188e-06, + "learning_rate": 2.8974605868602488e-05, + "loss": 1.7983825273404364e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1458, + "tokens/total": 22013748, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 471347 + }, + { + "epoch": 22.796875, + "grad_norm": 1.843075551732909e-05, + "learning_rate": 2.8915374541593737e-05, + "loss": 3.985167609243945e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1459, + "tokens/total": 22028824, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 471669 + }, + { + "epoch": 22.8125, + "grad_norm": 2.2321162759908475e-05, + "learning_rate": 2.8856211197312432e-05, + "loss": 3.975405604705884e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1460, + "tokens/total": 22044072, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 471993 + }, + { + "epoch": 22.828125, + "grad_norm": 2.928677713498473e-05, + "learning_rate": 2.8797115989952305e-05, + "loss": 5.932183739787433e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1461, + "tokens/total": 22059212, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 472310 + }, + { + "epoch": 22.84375, + "grad_norm": 3.648714846349321e-05, + "learning_rate": 2.873808907352949e-05, + "loss": 5.47908996395563e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1462, + "tokens/total": 22074312, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 472635 + }, + { + "epoch": 22.859375, + "grad_norm": 3.26577210216783e-05, + "learning_rate": 2.8679130601882144e-05, + "loss": 5.860315468453337e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1463, + "tokens/total": 22089440, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 472956 + }, + { + "epoch": 22.875, + "grad_norm": 1.780794445949141e-05, + "learning_rate": 2.862024072867007e-05, + "loss": 3.595536952616385e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1464, + "tokens/total": 22104464, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 473282 + }, + { + "epoch": 22.890625, + "grad_norm": 1.9158162103849463e-05, + "learning_rate": 2.8561419607374214e-05, + "loss": 4.05140696102535e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1465, + "tokens/total": 22119676, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 473608 + }, + { + "epoch": 22.90625, + "grad_norm": 6.942535947018769e-06, + "learning_rate": 2.8502667391296435e-05, + "loss": 1.3278763333346433e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1466, + "tokens/total": 22134644, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 473930 + }, + { + "epoch": 22.921875, + "grad_norm": 1.5135131434362847e-05, + "learning_rate": 2.8443984233558907e-05, + "loss": 3.3306270097455126e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1467, + "tokens/total": 22149764, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 474252 + }, + { + "epoch": 22.9375, + "grad_norm": 1.4279633433034178e-05, + "learning_rate": 2.838537028710394e-05, + "loss": 2.9391142675194715e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1468, + "tokens/total": 22164880, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 474579 + }, + { + "epoch": 22.953125, + "grad_norm": 1.023269123834325e-05, + "learning_rate": 2.8326825704693316e-05, + "loss": 2.188644998568634e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1469, + "tokens/total": 22180092, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 474902 + }, + { + "epoch": 22.96875, + "grad_norm": 1.4839905816188548e-05, + "learning_rate": 2.8268350638908182e-05, + "loss": 3.1993266702556866e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1470, + "tokens/total": 22195132, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 475225 + }, + { + "epoch": 22.984375, + "grad_norm": 5.386482735048048e-05, + "learning_rate": 2.8209945242148395e-05, + "loss": 8.787737328930234e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1471, + "tokens/total": 22210088, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 475547 + }, + { + "epoch": 23.0, + "grad_norm": 1.4094300240685698e-05, + "learning_rate": 2.815160966663235e-05, + "loss": 3.277225175679632e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1472, + "tokens/total": 22225280, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 475870 + }, + { + "epoch": 23.015625, + "grad_norm": 1.4401045518752653e-05, + "learning_rate": 2.809334406439633e-05, + "loss": 2.6827422061614925e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1473, + "tokens/total": 22240364, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 476193 + }, + { + "epoch": 23.03125, + "grad_norm": 1.6591920939390548e-05, + "learning_rate": 2.8035148587294387e-05, + "loss": 3.629593834375555e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1474, + "tokens/total": 22255444, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 476515 + }, + { + "epoch": 23.046875, + "grad_norm": 9.822988431551494e-06, + "learning_rate": 2.7977023386997735e-05, + "loss": 2.0877763518001302e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1475, + "tokens/total": 22270416, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 476841 + }, + { + "epoch": 23.0625, + "grad_norm": 1.2615361811185721e-05, + "learning_rate": 2.7918968614994445e-05, + "loss": 2.711046533931949e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1476, + "tokens/total": 22285748, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 477164 + }, + { + "epoch": 23.078125, + "grad_norm": 1.2093108125554863e-05, + "learning_rate": 2.7860984422589016e-05, + "loss": 2.5777751488931244e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1477, + "tokens/total": 22300808, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 477490 + }, + { + "epoch": 23.09375, + "grad_norm": 2.587634662631899e-05, + "learning_rate": 2.7803070960902062e-05, + "loss": 6.253110313991783e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1478, + "tokens/total": 22315984, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 477814 + }, + { + "epoch": 23.109375, + "grad_norm": 1.5266768969013356e-05, + "learning_rate": 2.7745228380869798e-05, + "loss": 3.1525311783298093e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1479, + "tokens/total": 22331044, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 478136 + }, + { + "epoch": 23.125, + "grad_norm": 1.5136975889618043e-05, + "learning_rate": 2.768745683324372e-05, + "loss": 2.426025673685217e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1480, + "tokens/total": 22346100, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 478457 + }, + { + "epoch": 23.140625, + "grad_norm": 2.0534154828055762e-05, + "learning_rate": 2.762975646859019e-05, + "loss": 4.143210219353932e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1481, + "tokens/total": 22361264, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 478778 + }, + { + "epoch": 23.15625, + "grad_norm": 0.00018273141176905483, + "learning_rate": 2.7572127437290053e-05, + "loss": 4.751656490498135e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1482, + "tokens/total": 22376396, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 479103 + }, + { + "epoch": 23.171875, + "grad_norm": 3.4000753657892346e-05, + "learning_rate": 2.7514569889538278e-05, + "loss": 5.095038204672164e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1483, + "tokens/total": 22391520, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 479426 + }, + { + "epoch": 23.1875, + "grad_norm": 1.8132228433387354e-05, + "learning_rate": 2.745708397534348e-05, + "loss": 4.1824580421234714e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1484, + "tokens/total": 22406480, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 479748 + }, + { + "epoch": 23.203125, + "grad_norm": 1.6882837371667847e-05, + "learning_rate": 2.7399669844527613e-05, + "loss": 2.4207386672969733e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1485, + "tokens/total": 22421664, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 480073 + }, + { + "epoch": 23.21875, + "grad_norm": 5.693377897841856e-05, + "learning_rate": 2.7342327646725506e-05, + "loss": 7.645189157301502e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1486, + "tokens/total": 22436748, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 480396 + }, + { + "epoch": 23.234375, + "grad_norm": 1.3248140930954833e-05, + "learning_rate": 2.7285057531384595e-05, + "loss": 2.9851560157112544e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1487, + "tokens/total": 22451728, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 480720 + }, + { + "epoch": 23.25, + "grad_norm": 2.221551949332934e-05, + "learning_rate": 2.722785964776433e-05, + "loss": 2.9605809004351613e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1488, + "tokens/total": 22466760, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 481042 + }, + { + "epoch": 23.265625, + "grad_norm": 1.6652656995574944e-05, + "learning_rate": 2.7170734144936028e-05, + "loss": 3.28441359442877e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1489, + "tokens/total": 22481856, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 481368 + }, + { + "epoch": 23.28125, + "grad_norm": 1.6585157936788164e-05, + "learning_rate": 2.711368117178228e-05, + "loss": 3.665095391625073e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1490, + "tokens/total": 22497004, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 481696 + }, + { + "epoch": 23.296875, + "grad_norm": 1.331805015070131e-05, + "learning_rate": 2.7056700876996732e-05, + "loss": 2.421159877030732e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1491, + "tokens/total": 22512116, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 482018 + }, + { + "epoch": 23.3125, + "grad_norm": 4.399784666020423e-05, + "learning_rate": 2.6999793409083496e-05, + "loss": 4.849848096455389e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1492, + "tokens/total": 22527256, + "tokens/train_per_sec_per_gpu": 9.68, + "tokens/trainable": 482344 + }, + { + "epoch": 23.328125, + "grad_norm": 1.5658615666325204e-05, + "learning_rate": 2.6942958916356998e-05, + "loss": 2.5000326786539517e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1493, + "tokens/total": 22542476, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 482670 + }, + { + "epoch": 23.34375, + "grad_norm": 1.8995611753780395e-05, + "learning_rate": 2.6886197546941383e-05, + "loss": 3.7242045891616726e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1494, + "tokens/total": 22557568, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 482987 + }, + { + "epoch": 23.359375, + "grad_norm": 3.240269870730117e-05, + "learning_rate": 2.6829509448770296e-05, + "loss": 5.720538638342987e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1495, + "tokens/total": 22572848, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 483312 + }, + { + "epoch": 23.375, + "grad_norm": 6.188770385051612e-06, + "learning_rate": 2.6772894769586332e-05, + "loss": 1.1260066656859635e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.0, + "step": 1496, + "tokens/total": 22588016, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 483638 + }, + { + "epoch": 23.390625, + "grad_norm": 2.7565496566239744e-05, + "learning_rate": 2.671635365694083e-05, + "loss": 5.212875748838997e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1497, + "tokens/total": 22603080, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 483963 + }, + { + "epoch": 23.40625, + "grad_norm": 2.707548628677614e-05, + "learning_rate": 2.665988625819334e-05, + "loss": 5.93018398831191e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1498, + "tokens/total": 22618280, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 484287 + }, + { + "epoch": 23.421875, + "grad_norm": 9.286595741286874e-06, + "learning_rate": 2.6603492720511312e-05, + "loss": 2.0135543365995545e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1499, + "tokens/total": 22633228, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 484612 + }, + { + "epoch": 23.4375, + "grad_norm": 0.00018082490714732558, + "learning_rate": 2.654717319086969e-05, + "loss": 1.6769070043665124e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1500, + "tokens/total": 22648180, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 484938 + }, + { + "epoch": 23.453125, + "grad_norm": 1.295412403123919e-05, + "learning_rate": 2.6490927816050526e-05, + "loss": 2.927469040514552e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1501, + "tokens/total": 22663176, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 485259 + }, + { + "epoch": 23.46875, + "grad_norm": 1.5645975508959964e-05, + "learning_rate": 2.643475674264265e-05, + "loss": 2.217900458845179e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1502, + "tokens/total": 22678328, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 485585 + }, + { + "epoch": 23.484375, + "grad_norm": 1.7083224520320073e-05, + "learning_rate": 2.637866011704121e-05, + "loss": 3.8717507777619176e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1503, + "tokens/total": 22693424, + "tokens/train_per_sec_per_gpu": 10.21, + "tokens/trainable": 485913 + }, + { + "epoch": 23.5, + "grad_norm": 2.7285470423521474e-05, + "learning_rate": 2.6322638085447334e-05, + "loss": 5.475638431562402e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1504, + "tokens/total": 22708616, + "tokens/train_per_sec_per_gpu": 9.0, + "tokens/trainable": 486235 + }, + { + "epoch": 23.515625, + "grad_norm": 3.456019112491049e-05, + "learning_rate": 2.626669079386771e-05, + "loss": 6.880909495521337e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1505, + "tokens/total": 22723648, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 486557 + }, + { + "epoch": 23.53125, + "grad_norm": 9.22158233152004e-06, + "learning_rate": 2.62108183881143e-05, + "loss": 1.9060682632243697e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1506, + "tokens/total": 22738744, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 486879 + }, + { + "epoch": 23.546875, + "grad_norm": 1.173120199382538e-05, + "learning_rate": 2.6155021013803842e-05, + "loss": 2.6135080588574056e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1507, + "tokens/total": 22753912, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 487203 + }, + { + "epoch": 23.5625, + "grad_norm": 1.1295921467535663e-05, + "learning_rate": 2.6099298816357555e-05, + "loss": 2.428663776754547e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1508, + "tokens/total": 22769040, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 487527 + }, + { + "epoch": 23.578125, + "grad_norm": 1.7376589312334545e-05, + "learning_rate": 2.604365194100068e-05, + "loss": 3.7264086927280005e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1509, + "tokens/total": 22784012, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 487849 + }, + { + "epoch": 23.59375, + "grad_norm": 4.102769889868796e-05, + "learning_rate": 2.5988080532762248e-05, + "loss": 7.02444594935514e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1510, + "tokens/total": 22799172, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 488172 + }, + { + "epoch": 23.609375, + "grad_norm": 2.1342699255910702e-05, + "learning_rate": 2.593258473647448e-05, + "loss": 3.0051018029553234e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1511, + "tokens/total": 22814124, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 488495 + }, + { + "epoch": 23.625, + "grad_norm": 2.7436250093160197e-05, + "learning_rate": 2.5877164696772638e-05, + "loss": 5.514424969987886e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1512, + "tokens/total": 22829148, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 488817 + }, + { + "epoch": 23.640625, + "grad_norm": 3.294597627245821e-05, + "learning_rate": 2.5821820558094483e-05, + "loss": 5.691269961971557e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1513, + "tokens/total": 22844272, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 489141 + }, + { + "epoch": 23.65625, + "grad_norm": 3.386152093298733e-05, + "learning_rate": 2.5766552464680015e-05, + "loss": 5.923816956965311e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1514, + "tokens/total": 22859352, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 489465 + }, + { + "epoch": 23.671875, + "grad_norm": 1.3713202861254103e-05, + "learning_rate": 2.571136056057096e-05, + "loss": 2.6153662702199654e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1515, + "tokens/total": 22874376, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 489789 + }, + { + "epoch": 23.6875, + "grad_norm": 8.64681278471835e-05, + "learning_rate": 2.5656244989610568e-05, + "loss": 1.2359934089545277e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1516, + "tokens/total": 22889344, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 490112 + }, + { + "epoch": 23.703125, + "grad_norm": 1.9240829715272412e-05, + "learning_rate": 2.560120589544308e-05, + "loss": 4.0958579461403133e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1517, + "tokens/total": 22904408, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 490437 + }, + { + "epoch": 23.71875, + "grad_norm": 2.2422669644583948e-05, + "learning_rate": 2.5546243421513444e-05, + "loss": 3.552950147422962e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1518, + "tokens/total": 22919460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 490756 + }, + { + "epoch": 23.734375, + "grad_norm": 2.406825660727918e-05, + "learning_rate": 2.549135771106693e-05, + "loss": 2.971675598928414e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1519, + "tokens/total": 22934492, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 491085 + }, + { + "epoch": 23.75, + "grad_norm": 0.00011643066682154313, + "learning_rate": 2.5436548907148706e-05, + "loss": 1.1498046887936653e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1520, + "tokens/total": 22949552, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 491406 + }, + { + "epoch": 23.765625, + "grad_norm": 1.5036646800581366e-05, + "learning_rate": 2.538181715260356e-05, + "loss": 3.551491829512088e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1521, + "tokens/total": 22964652, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 491728 + }, + { + "epoch": 23.78125, + "grad_norm": 1.697290463198442e-05, + "learning_rate": 2.5327162590075426e-05, + "loss": 3.544531637089676e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1522, + "tokens/total": 22979728, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 492051 + }, + { + "epoch": 23.796875, + "grad_norm": 3.9244005165528506e-05, + "learning_rate": 2.5272585362007067e-05, + "loss": 8.17311672562937e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1523, + "tokens/total": 22994732, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 492372 + }, + { + "epoch": 23.8125, + "grad_norm": 2.9165606974856928e-05, + "learning_rate": 2.5218085610639696e-05, + "loss": 3.387165179447038e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1524, + "tokens/total": 23009844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 492696 + }, + { + "epoch": 23.828125, + "grad_norm": 3.211217699572444e-05, + "learning_rate": 2.5163663478012618e-05, + "loss": 3.639808028310654e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1525, + "tokens/total": 23025024, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 493021 + }, + { + "epoch": 23.84375, + "grad_norm": 2.481511364749167e-05, + "learning_rate": 2.5109319105962835e-05, + "loss": 5.799474820378236e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1526, + "tokens/total": 23040172, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 493343 + }, + { + "epoch": 23.859375, + "grad_norm": 1.5476774933631532e-05, + "learning_rate": 2.5055052636124666e-05, + "loss": 3.0946594620218093e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1527, + "tokens/total": 23055144, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 493666 + }, + { + "epoch": 23.875, + "grad_norm": 2.0063158444827422e-05, + "learning_rate": 2.500086420992942e-05, + "loss": 4.508712549977645e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1528, + "tokens/total": 23070124, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 493987 + }, + { + "epoch": 23.890625, + "grad_norm": 1.668175718805287e-05, + "learning_rate": 2.4946753968605012e-05, + "loss": 3.341960734815075e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1529, + "tokens/total": 23085176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 494304 + }, + { + "epoch": 23.90625, + "grad_norm": 1.7266431314055808e-05, + "learning_rate": 2.4892722053175583e-05, + "loss": 3.93318458691283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1530, + "tokens/total": 23100172, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 494624 + }, + { + "epoch": 23.921875, + "grad_norm": 1.0495174137759022e-05, + "learning_rate": 2.4838768604461125e-05, + "loss": 2.235168068409621e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1531, + "tokens/total": 23115288, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 494946 + }, + { + "epoch": 23.9375, + "grad_norm": 1.6368403521482833e-05, + "learning_rate": 2.4784893763077116e-05, + "loss": 3.0361857739080733e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1532, + "tokens/total": 23130428, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 495269 + }, + { + "epoch": 23.953125, + "grad_norm": 1.5549949239357375e-05, + "learning_rate": 2.473109766943422e-05, + "loss": 3.4983861496584723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1533, + "tokens/total": 23145600, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 495592 + }, + { + "epoch": 23.96875, + "grad_norm": 2.542834590713028e-05, + "learning_rate": 2.467738046373782e-05, + "loss": 3.588768890949723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1534, + "tokens/total": 23160692, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 495915 + }, + { + "epoch": 23.984375, + "grad_norm": 2.9186734536779113e-05, + "learning_rate": 2.4623742285987702e-05, + "loss": 6.73559384267719e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1535, + "tokens/total": 23175800, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 496239 + }, + { + "epoch": 24.0, + "grad_norm": 1.6156320270965807e-05, + "learning_rate": 2.4570183275977694e-05, + "loss": 2.62379330706608e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1536, + "tokens/total": 23190820, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 496560 + }, + { + "epoch": 24.015625, + "grad_norm": 0.00018328192527405918, + "learning_rate": 2.4516703573295287e-05, + "loss": 1.9487943063722923e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1537, + "tokens/total": 23205920, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 496882 + }, + { + "epoch": 24.03125, + "grad_norm": 1.4727082088938914e-05, + "learning_rate": 2.4463303317321305e-05, + "loss": 3.2395499260928773e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1538, + "tokens/total": 23220964, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 497207 + }, + { + "epoch": 24.046875, + "grad_norm": 9.97033748717513e-06, + "learning_rate": 2.4409982647229465e-05, + "loss": 1.9600727796387218e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1539, + "tokens/total": 23236064, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 497538 + }, + { + "epoch": 24.0625, + "grad_norm": 3.534083953127265e-05, + "learning_rate": 2.4356741701986123e-05, + "loss": 6.227217568266497e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1540, + "tokens/total": 23251248, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 497859 + }, + { + "epoch": 24.078125, + "grad_norm": 1.0700529855967034e-05, + "learning_rate": 2.4303580620349787e-05, + "loss": 2.0569157754835032e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1541, + "tokens/total": 23266332, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 498180 + }, + { + "epoch": 24.09375, + "grad_norm": 0.00012721343955490738, + "learning_rate": 2.4250499540870918e-05, + "loss": 1.2804268862964818e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1542, + "tokens/total": 23281472, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 498497 + }, + { + "epoch": 24.109375, + "grad_norm": 1.023274762701476e-05, + "learning_rate": 2.4197498601891344e-05, + "loss": 2.2081765393977548e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1543, + "tokens/total": 23296536, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 498822 + }, + { + "epoch": 24.125, + "grad_norm": 1.1989674931101035e-05, + "learning_rate": 2.4144577941544145e-05, + "loss": 2.2705900448727334e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1544, + "tokens/total": 23311696, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 499148 + }, + { + "epoch": 24.140625, + "grad_norm": 7.195690614025807e-06, + "learning_rate": 2.4091737697753107e-05, + "loss": 1.5436896205756057e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1545, + "tokens/total": 23326708, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 499474 + }, + { + "epoch": 24.15625, + "grad_norm": 1.697172410786152e-05, + "learning_rate": 2.403897800823246e-05, + "loss": 2.9672207801922923e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1546, + "tokens/total": 23341868, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 499797 + }, + { + "epoch": 24.171875, + "grad_norm": 2.2683390852762386e-05, + "learning_rate": 2.3986299010486463e-05, + "loss": 4.587024875490897e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1547, + "tokens/total": 23357088, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 500125 + }, + { + "epoch": 24.1875, + "grad_norm": 2.3140886696637608e-05, + "learning_rate": 2.393370084180912e-05, + "loss": 3.5734481684812636e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1548, + "tokens/total": 23372184, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 500451 + }, + { + "epoch": 24.203125, + "grad_norm": 3.098443994531408e-05, + "learning_rate": 2.3881183639283732e-05, + "loss": 2.5058935193555953e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1549, + "tokens/total": 23387316, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 500769 + }, + { + "epoch": 24.21875, + "grad_norm": 2.092586328217294e-05, + "learning_rate": 2.3828747539782608e-05, + "loss": 4.880834580944793e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1550, + "tokens/total": 23402500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 501086 + }, + { + "epoch": 24.234375, + "grad_norm": 3.14128483296372e-05, + "learning_rate": 2.3776392679966642e-05, + "loss": 3.9183740341286466e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1551, + "tokens/total": 23417520, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 501407 + }, + { + "epoch": 24.25, + "grad_norm": 1.8294416804565117e-05, + "learning_rate": 2.3724119196285073e-05, + "loss": 1.920086845075275e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1552, + "tokens/total": 23432652, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 501731 + }, + { + "epoch": 24.265625, + "grad_norm": 1.0029069017036818e-05, + "learning_rate": 2.367192722497499e-05, + "loss": 2.0357781238544703e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1553, + "tokens/total": 23447672, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 502052 + }, + { + "epoch": 24.28125, + "grad_norm": 3.8349600799847394e-05, + "learning_rate": 2.361981690206107e-05, + "loss": 5.110204028824228e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1554, + "tokens/total": 23462664, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 502377 + }, + { + "epoch": 24.296875, + "grad_norm": 2.5826582714216784e-05, + "learning_rate": 2.356778836335518e-05, + "loss": 5.55304382032773e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1555, + "tokens/total": 23477764, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 502705 + }, + { + "epoch": 24.3125, + "grad_norm": 1.2582670024130493e-05, + "learning_rate": 2.3515841744456036e-05, + "loss": 2.3305972263187869e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1556, + "tokens/total": 23492880, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 503027 + }, + { + "epoch": 24.328125, + "grad_norm": 3.239547731936909e-05, + "learning_rate": 2.3463977180748894e-05, + "loss": 7.487965376640204e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1557, + "tokens/total": 23507924, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 503353 + }, + { + "epoch": 24.34375, + "grad_norm": 2.7488897103467025e-05, + "learning_rate": 2.341219480740508e-05, + "loss": 5.454018037198693e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1558, + "tokens/total": 23522900, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 503672 + }, + { + "epoch": 24.359375, + "grad_norm": 1.0040050256066024e-05, + "learning_rate": 2.3360494759381786e-05, + "loss": 1.776407003717395e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1559, + "tokens/total": 23537884, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 504002 + }, + { + "epoch": 24.375, + "grad_norm": 9.075505658984184e-06, + "learning_rate": 2.330887717142159e-05, + "loss": 1.84155609872505e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1560, + "tokens/total": 23552944, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 504328 + }, + { + "epoch": 24.390625, + "grad_norm": 1.0076410035253502e-05, + "learning_rate": 2.3257342178052214e-05, + "loss": 2.057635413166281e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1561, + "tokens/total": 23568188, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 504653 + }, + { + "epoch": 24.40625, + "grad_norm": 1.061128295987146e-05, + "learning_rate": 2.3205889913586044e-05, + "loss": 2.306671120777537e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1562, + "tokens/total": 23583404, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 504980 + }, + { + "epoch": 24.421875, + "grad_norm": 4.271507350495085e-05, + "learning_rate": 2.3154520512119936e-05, + "loss": 7.778328381391475e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1563, + "tokens/total": 23598636, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 505300 + }, + { + "epoch": 24.4375, + "grad_norm": 6.207700789673254e-05, + "learning_rate": 2.3103234107534718e-05, + "loss": 7.89354317021207e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1564, + "tokens/total": 23613708, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 505627 + }, + { + "epoch": 24.453125, + "grad_norm": 1.117512056225678e-05, + "learning_rate": 2.3052030833494976e-05, + "loss": 2.265819318836293e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1565, + "tokens/total": 23628916, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 505952 + }, + { + "epoch": 24.46875, + "grad_norm": 2.684137325559277e-05, + "learning_rate": 2.3000910823448554e-05, + "loss": 3.9289176356760436e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1566, + "tokens/total": 23643948, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 506271 + }, + { + "epoch": 24.484375, + "grad_norm": 1.3400883290159982e-05, + "learning_rate": 2.294987421062637e-05, + "loss": 2.596192132386932e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1567, + "tokens/total": 23659212, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 506598 + }, + { + "epoch": 24.5, + "grad_norm": 1.8385188013780862e-05, + "learning_rate": 2.2898921128041928e-05, + "loss": 3.4672405035962583e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1568, + "tokens/total": 23674332, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 506922 + }, + { + "epoch": 24.515625, + "grad_norm": 1.7855074474937283e-05, + "learning_rate": 2.2848051708491098e-05, + "loss": 1.6785767797955486e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1569, + "tokens/total": 23689316, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 507246 + }, + { + "epoch": 24.53125, + "grad_norm": 1.671212521614507e-05, + "learning_rate": 2.279726608455162e-05, + "loss": 2.951998965272651e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1570, + "tokens/total": 23704384, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 507566 + }, + { + "epoch": 24.546875, + "grad_norm": 1.2489974324125797e-05, + "learning_rate": 2.2746564388582926e-05, + "loss": 2.8516868155747943e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1571, + "tokens/total": 23719504, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 507890 + }, + { + "epoch": 24.5625, + "grad_norm": 1.6493149814778008e-05, + "learning_rate": 2.2695946752725654e-05, + "loss": 3.6613045040212455e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1572, + "tokens/total": 23734576, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 508213 + }, + { + "epoch": 24.578125, + "grad_norm": 2.65638100245269e-05, + "learning_rate": 2.2645413308901387e-05, + "loss": 4.6345357418431377e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1573, + "tokens/total": 23749596, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 508529 + }, + { + "epoch": 24.59375, + "grad_norm": 2.3676291675656103e-05, + "learning_rate": 2.259496418881228e-05, + "loss": 4.273466913673474e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1574, + "tokens/total": 23764704, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 508852 + }, + { + "epoch": 24.609375, + "grad_norm": 2.3414606403093785e-05, + "learning_rate": 2.2544599523940698e-05, + "loss": 5.091196726425551e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1575, + "tokens/total": 23779744, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 509176 + }, + { + "epoch": 24.625, + "grad_norm": 2.8096381356590427e-05, + "learning_rate": 2.249431944554895e-05, + "loss": 5.013213808524597e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1576, + "tokens/total": 23794880, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 509501 + }, + { + "epoch": 24.640625, + "grad_norm": 1.4321371963887941e-05, + "learning_rate": 2.2444124084678846e-05, + "loss": 2.920194219768746e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1577, + "tokens/total": 23809888, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 509832 + }, + { + "epoch": 24.65625, + "grad_norm": 2.7232124921283685e-05, + "learning_rate": 2.239401357215141e-05, + "loss": 4.700288229742e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1578, + "tokens/total": 23825052, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 510161 + }, + { + "epoch": 24.671875, + "grad_norm": 1.824104765546508e-05, + "learning_rate": 2.2343988038566527e-05, + "loss": 3.6806969205827045e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1579, + "tokens/total": 23840176, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 510487 + }, + { + "epoch": 24.6875, + "grad_norm": 1.4824463505647145e-05, + "learning_rate": 2.2294047614302638e-05, + "loss": 2.9522342970267346e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1580, + "tokens/total": 23855280, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 510808 + }, + { + "epoch": 24.703125, + "grad_norm": 2.8929784093634225e-05, + "learning_rate": 2.224419242951633e-05, + "loss": 4.5896260303379677e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1581, + "tokens/total": 23870400, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 511126 + }, + { + "epoch": 24.71875, + "grad_norm": 1.4083047062740661e-05, + "learning_rate": 2.2194422614142064e-05, + "loss": 3.079419172991038e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1582, + "tokens/total": 23885624, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 511447 + }, + { + "epoch": 24.734375, + "grad_norm": 1.8516500858822837e-05, + "learning_rate": 2.214473829789177e-05, + "loss": 4.115583749353391e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1583, + "tokens/total": 23900688, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 511768 + }, + { + "epoch": 24.75, + "grad_norm": 9.315471106674522e-05, + "learning_rate": 2.2095139610254622e-05, + "loss": 1.407270929121296e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1584, + "tokens/total": 23915852, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 512090 + }, + { + "epoch": 24.765625, + "grad_norm": 8.486517799610738e-06, + "learning_rate": 2.204562668049652e-05, + "loss": 1.7648183359142422e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1585, + "tokens/total": 23930944, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 512414 + }, + { + "epoch": 24.78125, + "grad_norm": 0.00012215440801810473, + "learning_rate": 2.1996199637659957e-05, + "loss": 4.3897458112951426e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1586, + "tokens/total": 23946024, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 512732 + }, + { + "epoch": 24.796875, + "grad_norm": 6.840926744189346e-06, + "learning_rate": 2.194685861056351e-05, + "loss": 1.3592912750937103e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1587, + "tokens/total": 23960988, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 513054 + }, + { + "epoch": 24.8125, + "grad_norm": 1.7666594430920668e-05, + "learning_rate": 2.1897603727801656e-05, + "loss": 3.826436909548647e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1588, + "tokens/total": 23975988, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 513379 + }, + { + "epoch": 24.828125, + "grad_norm": 2.3360762497759424e-05, + "learning_rate": 2.1848435117744266e-05, + "loss": 3.701095465658e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1589, + "tokens/total": 23991136, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 513703 + }, + { + "epoch": 24.84375, + "grad_norm": 1.1754115803341847e-05, + "learning_rate": 2.1799352908536442e-05, + "loss": 2.5665221414783446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1590, + "tokens/total": 24006212, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 514023 + }, + { + "epoch": 24.859375, + "grad_norm": 2.3168613552115858e-05, + "learning_rate": 2.175035722809806e-05, + "loss": 5.177637945052993e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1591, + "tokens/total": 24021360, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 514345 + }, + { + "epoch": 24.875, + "grad_norm": 2.8228480005054735e-05, + "learning_rate": 2.1701448204123515e-05, + "loss": 5.612337190541439e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1592, + "tokens/total": 24036532, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 514665 + }, + { + "epoch": 24.890625, + "grad_norm": 1.5557567166979425e-05, + "learning_rate": 2.165262596408132e-05, + "loss": 3.153788270537916e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1593, + "tokens/total": 24051756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 514991 + }, + { + "epoch": 24.90625, + "grad_norm": 1.0747134183475282e-05, + "learning_rate": 2.160389063521382e-05, + "loss": 2.337792608386735e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1594, + "tokens/total": 24066808, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 515317 + }, + { + "epoch": 24.921875, + "grad_norm": 1.5238619198498782e-05, + "learning_rate": 2.1555242344536873e-05, + "loss": 3.233716654449381e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1595, + "tokens/total": 24082064, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 515646 + }, + { + "epoch": 24.9375, + "grad_norm": 1.1567119145183824e-05, + "learning_rate": 2.150668121883947e-05, + "loss": 2.2523538234509033e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1596, + "tokens/total": 24097152, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 515965 + }, + { + "epoch": 24.953125, + "grad_norm": 9.870591384242289e-06, + "learning_rate": 2.1458207384683428e-05, + "loss": 1.590055092037801e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1597, + "tokens/total": 24112152, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 516286 + }, + { + "epoch": 24.96875, + "grad_norm": 1.7431369997211732e-05, + "learning_rate": 2.1409820968403056e-05, + "loss": 3.8685413983330363e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1598, + "tokens/total": 24127136, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 516605 + }, + { + "epoch": 24.984375, + "grad_norm": 3.312345143058337e-05, + "learning_rate": 2.1361522096104875e-05, + "loss": 5.133285867486848e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1599, + "tokens/total": 24142308, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 516925 + }, + { + "epoch": 25.0, + "grad_norm": 1.5798903405084275e-05, + "learning_rate": 2.1313310893667192e-05, + "loss": 3.3984682090704155e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1600, + "tokens/total": 24157356, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 517250 + }, + { + "epoch": 25.015625, + "grad_norm": 4.2902178392978385e-05, + "learning_rate": 2.126518748673986e-05, + "loss": 2.731882204898284e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1601, + "tokens/total": 24172344, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 517575 + }, + { + "epoch": 25.03125, + "grad_norm": 0.0001015453235595487, + "learning_rate": 2.1217152000743884e-05, + "loss": 1.5506770978390705e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1602, + "tokens/total": 24187484, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 517903 + }, + { + "epoch": 25.046875, + "grad_norm": 2.575152029749006e-05, + "learning_rate": 2.1169204560871172e-05, + "loss": 1.7059325330137654e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1603, + "tokens/total": 24202512, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 518233 + }, + { + "epoch": 25.0625, + "grad_norm": 3.0647188395960256e-05, + "learning_rate": 2.1121345292084127e-05, + "loss": 3.457152502051031e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1604, + "tokens/total": 24217588, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 518558 + }, + { + "epoch": 25.078125, + "grad_norm": 0.00011346596875227988, + "learning_rate": 2.1073574319115363e-05, + "loss": 9.643015346227912e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1605, + "tokens/total": 24232700, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 518881 + }, + { + "epoch": 25.09375, + "grad_norm": 1.5182874449237715e-05, + "learning_rate": 2.102589176646736e-05, + "loss": 3.1043919079820625e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1606, + "tokens/total": 24247884, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 519205 + }, + { + "epoch": 25.109375, + "grad_norm": 1.9169516235706396e-05, + "learning_rate": 2.0978297758412214e-05, + "loss": 3.848487608593132e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1607, + "tokens/total": 24262984, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 519530 + }, + { + "epoch": 25.125, + "grad_norm": 2.6819219783646986e-05, + "learning_rate": 2.0930792418991184e-05, + "loss": 6.578692364200833e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1608, + "tokens/total": 24278048, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 519853 + }, + { + "epoch": 25.140625, + "grad_norm": 1.4344253031595144e-05, + "learning_rate": 2.088337587201447e-05, + "loss": 2.8448386046875385e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1609, + "tokens/total": 24293232, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 520178 + }, + { + "epoch": 25.15625, + "grad_norm": 2.1109824956511147e-05, + "learning_rate": 2.083604824106085e-05, + "loss": 4.580232371154125e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1610, + "tokens/total": 24308332, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 520503 + }, + { + "epoch": 25.171875, + "grad_norm": 5.620383672066964e-05, + "learning_rate": 2.078880964947736e-05, + "loss": 8.826678481455019e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1611, + "tokens/total": 24323444, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 520825 + }, + { + "epoch": 25.1875, + "grad_norm": 2.3172660803538747e-05, + "learning_rate": 2.0741660220379012e-05, + "loss": 4.091206164957839e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1612, + "tokens/total": 24338496, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 521149 + }, + { + "epoch": 25.203125, + "grad_norm": 1.075820364349056e-05, + "learning_rate": 2.069460007664838e-05, + "loss": 2.127154630215955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1613, + "tokens/total": 24353644, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 521469 + }, + { + "epoch": 25.21875, + "grad_norm": 0.0001669779885560274, + "learning_rate": 2.0647629340935387e-05, + "loss": 1.5793682450748747e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1614, + "tokens/total": 24368700, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 521797 + }, + { + "epoch": 25.234375, + "grad_norm": 6.74806733513833e-06, + "learning_rate": 2.0600748135656926e-05, + "loss": 1.3862594983038434e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1615, + "tokens/total": 24383768, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 522116 + }, + { + "epoch": 25.25, + "grad_norm": 1.440322921553161e-05, + "learning_rate": 2.055395658299656e-05, + "loss": 2.917277015512809e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1616, + "tokens/total": 24398892, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 522437 + }, + { + "epoch": 25.265625, + "grad_norm": 1.946500560734421e-05, + "learning_rate": 2.0507254804904158e-05, + "loss": 3.7510565675802354e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1617, + "tokens/total": 24413896, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 522756 + }, + { + "epoch": 25.28125, + "grad_norm": 1.2273356333025731e-05, + "learning_rate": 2.0460642923095662e-05, + "loss": 2.4442454105155775e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1618, + "tokens/total": 24429044, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 523075 + }, + { + "epoch": 25.296875, + "grad_norm": 1.0969366485369392e-05, + "learning_rate": 2.0414121059052703e-05, + "loss": 2.364093205642348e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1619, + "tokens/total": 24444152, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 523399 + }, + { + "epoch": 25.3125, + "grad_norm": 1.8300721421837807e-05, + "learning_rate": 2.03676893340223e-05, + "loss": 2.889397308081243e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1620, + "tokens/total": 24459260, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 523720 + }, + { + "epoch": 25.328125, + "grad_norm": 1.2247912309248932e-05, + "learning_rate": 2.0321347869016542e-05, + "loss": 2.0875539519238373e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1621, + "tokens/total": 24474312, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 524049 + }, + { + "epoch": 25.34375, + "grad_norm": 2.7456364477984607e-05, + "learning_rate": 2.0275096784812313e-05, + "loss": 5.183035227673827e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1622, + "tokens/total": 24489468, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 524376 + }, + { + "epoch": 25.359375, + "grad_norm": 2.2560099750990048e-05, + "learning_rate": 2.022893620195091e-05, + "loss": 3.8582373917961377e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1623, + "tokens/total": 24504508, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 524695 + }, + { + "epoch": 25.375, + "grad_norm": 1.0694227967178449e-05, + "learning_rate": 2.0182866240737768e-05, + "loss": 2.5947718995666946e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1624, + "tokens/total": 24519640, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 525026 + }, + { + "epoch": 25.390625, + "grad_norm": 1.949435318238102e-05, + "learning_rate": 2.013688702124214e-05, + "loss": 3.9661330220042146e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1625, + "tokens/total": 24534840, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 525348 + }, + { + "epoch": 25.40625, + "grad_norm": 1.009213156066835e-05, + "learning_rate": 2.0090998663296813e-05, + "loss": 1.9991482247405656e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1626, + "tokens/total": 24549984, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 525672 + }, + { + "epoch": 25.421875, + "grad_norm": 1.716102269710973e-05, + "learning_rate": 2.0045201286497722e-05, + "loss": 3.7524287677115353e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1627, + "tokens/total": 24565016, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 525997 + }, + { + "epoch": 25.4375, + "grad_norm": 1.6276855603791773e-05, + "learning_rate": 1.9999495010203705e-05, + "loss": 3.588227741602168e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1628, + "tokens/total": 24580144, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 526320 + }, + { + "epoch": 25.453125, + "grad_norm": 1.723960303934291e-05, + "learning_rate": 1.9953879953536188e-05, + "loss": 2.9263037504279055e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1629, + "tokens/total": 24595228, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 526645 + }, + { + "epoch": 25.46875, + "grad_norm": 3.68501350749284e-05, + "learning_rate": 1.9908356235378808e-05, + "loss": 4.792572667611239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1630, + "tokens/total": 24610336, + "tokens/train_per_sec_per_gpu": 10.0, + "tokens/trainable": 526970 + }, + { + "epoch": 25.484375, + "grad_norm": 2.428412699373439e-05, + "learning_rate": 1.9862923974377228e-05, + "loss": 3.53087585835965e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1631, + "tokens/total": 24625392, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 527297 + }, + { + "epoch": 25.5, + "grad_norm": 8.640634405310266e-06, + "learning_rate": 1.981758328893866e-05, + "loss": 1.8481665620129206e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1632, + "tokens/total": 24640360, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 527620 + }, + { + "epoch": 25.515625, + "grad_norm": 1.2903607967018615e-05, + "learning_rate": 1.9772334297231732e-05, + "loss": 2.504151552784606e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1633, + "tokens/total": 24655468, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 527942 + }, + { + "epoch": 25.53125, + "grad_norm": 1.010856794891879e-05, + "learning_rate": 1.972717711718604e-05, + "loss": 1.8087268927047262e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1634, + "tokens/total": 24670420, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 528267 + }, + { + "epoch": 25.546875, + "grad_norm": 3.027995626325719e-05, + "learning_rate": 1.9682111866491943e-05, + "loss": 5.604180159934913e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1635, + "tokens/total": 24685548, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 528589 + }, + { + "epoch": 25.5625, + "grad_norm": 2.6431034712004475e-05, + "learning_rate": 1.963713866260014e-05, + "loss": 2.3369639734482917e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1636, + "tokens/total": 24700664, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 528915 + }, + { + "epoch": 25.578125, + "grad_norm": 2.9279766749823466e-05, + "learning_rate": 1.9592257622721517e-05, + "loss": 3.4665396242417046e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1637, + "tokens/total": 24715768, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 529239 + }, + { + "epoch": 25.59375, + "grad_norm": 1.6904110452742316e-05, + "learning_rate": 1.95474688638267e-05, + "loss": 2.87459670289536e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1638, + "tokens/total": 24730812, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 529559 + }, + { + "epoch": 25.609375, + "grad_norm": 2.7232688807998784e-05, + "learning_rate": 1.9502772502645857e-05, + "loss": 3.1733662808619556e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1639, + "tokens/total": 24746024, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 529884 + }, + { + "epoch": 25.625, + "grad_norm": 1.781438368197996e-05, + "learning_rate": 1.945816865566828e-05, + "loss": 3.6750628851223155e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1640, + "tokens/total": 24761172, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 530209 + }, + { + "epoch": 25.640625, + "grad_norm": 5.175339902052656e-05, + "learning_rate": 1.9413657439142208e-05, + "loss": 1.1526402659001178e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1641, + "tokens/total": 24776264, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 530530 + }, + { + "epoch": 25.65625, + "grad_norm": 1.0831482541107107e-05, + "learning_rate": 1.9369238969074445e-05, + "loss": 2.2932617582682724e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1642, + "tokens/total": 24791412, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 530853 + }, + { + "epoch": 25.671875, + "grad_norm": 1.0218879651802126e-05, + "learning_rate": 1.9324913361230052e-05, + "loss": 2.1647170456162712e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1643, + "tokens/total": 24806456, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 531175 + }, + { + "epoch": 25.6875, + "grad_norm": 1.4917302905814722e-05, + "learning_rate": 1.9280680731132082e-05, + "loss": 2.501063818272087e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1644, + "tokens/total": 24821460, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 531498 + }, + { + "epoch": 25.703125, + "grad_norm": 1.4758047655050177e-05, + "learning_rate": 1.923654119406131e-05, + "loss": 3.096523357726255e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1645, + "tokens/total": 24836548, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 531815 + }, + { + "epoch": 25.71875, + "grad_norm": 9.48030356084928e-06, + "learning_rate": 1.9192494865055816e-05, + "loss": 2.0583628668191523e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1646, + "tokens/total": 24851748, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 532140 + }, + { + "epoch": 25.734375, + "grad_norm": 1.776542376319412e-05, + "learning_rate": 1.9148541858910806e-05, + "loss": 3.955955492074281e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1647, + "tokens/total": 24866908, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 532463 + }, + { + "epoch": 25.75, + "grad_norm": 1.6392319594160654e-05, + "learning_rate": 1.9104682290178246e-05, + "loss": 3.635446432781464e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1648, + "tokens/total": 24881860, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 532786 + }, + { + "epoch": 25.765625, + "grad_norm": 2.2823722247267142e-05, + "learning_rate": 1.9060916273166573e-05, + "loss": 5.115059593663318e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1649, + "tokens/total": 24896916, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 533103 + }, + { + "epoch": 25.78125, + "grad_norm": 2.120976751029957e-05, + "learning_rate": 1.901724392194044e-05, + "loss": 3.726216561972251e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1650, + "tokens/total": 24912032, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 533428 + }, + { + "epoch": 25.796875, + "grad_norm": 1.3364609912969172e-05, + "learning_rate": 1.8973665350320364e-05, + "loss": 2.9316154837033537e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1651, + "tokens/total": 24927176, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 533755 + }, + { + "epoch": 25.8125, + "grad_norm": 3.055337947444059e-05, + "learning_rate": 1.8930180671882437e-05, + "loss": 4.090614424967498e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1652, + "tokens/total": 24942200, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 534078 + }, + { + "epoch": 25.828125, + "grad_norm": 1.9468014215817675e-05, + "learning_rate": 1.8886789999958054e-05, + "loss": 4.179453299002489e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1653, + "tokens/total": 24957280, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 534401 + }, + { + "epoch": 25.84375, + "grad_norm": 1.8113187252311036e-05, + "learning_rate": 1.8843493447633634e-05, + "loss": 4.092138965461345e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1654, + "tokens/total": 24972348, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 534724 + }, + { + "epoch": 25.859375, + "grad_norm": 1.4480810932582244e-05, + "learning_rate": 1.880029112775023e-05, + "loss": 2.4120743091771146e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1655, + "tokens/total": 24987556, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 535046 + }, + { + "epoch": 25.875, + "grad_norm": 2.254635182907805e-05, + "learning_rate": 1.8757183152903383e-05, + "loss": 3.2750455147834145e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1656, + "tokens/total": 25002600, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 535369 + }, + { + "epoch": 25.890625, + "grad_norm": 1.0972379641316365e-05, + "learning_rate": 1.8714169635442687e-05, + "loss": 2.264431202547712e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1657, + "tokens/total": 25017708, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 535690 + }, + { + "epoch": 25.90625, + "grad_norm": 1.5460858776350506e-05, + "learning_rate": 1.867125068747162e-05, + "loss": 3.4306842167097784e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1658, + "tokens/total": 25032856, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 536010 + }, + { + "epoch": 25.921875, + "grad_norm": 7.150162673497107e-06, + "learning_rate": 1.8628426420847094e-05, + "loss": 1.405391714115467e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1659, + "tokens/total": 25047956, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 536331 + }, + { + "epoch": 25.9375, + "grad_norm": 3.4945162042276934e-05, + "learning_rate": 1.8585696947179357e-05, + "loss": 2.2341077965393197e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1660, + "tokens/total": 25063100, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 536651 + }, + { + "epoch": 25.953125, + "grad_norm": 6.57899090583669e-06, + "learning_rate": 1.854306237783154e-05, + "loss": 1.2845691799157066e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1661, + "tokens/total": 25078168, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 536977 + }, + { + "epoch": 25.96875, + "grad_norm": 2.2198901206138544e-05, + "learning_rate": 1.850052282391949e-05, + "loss": 3.7974319866407313e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1662, + "tokens/total": 25093348, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 537296 + }, + { + "epoch": 25.984375, + "grad_norm": 1.1136278772028163e-05, + "learning_rate": 1.8458078396311336e-05, + "loss": 2.0265468947400223e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1663, + "tokens/total": 25108520, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 537619 + }, + { + "epoch": 26.0, + "grad_norm": 2.3501183022744954e-05, + "learning_rate": 1.8415729205627368e-05, + "loss": 3.9794144868210424e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1664, + "tokens/total": 25123596, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 537940 + }, + { + "epoch": 26.015625, + "grad_norm": 5.885937571292743e-05, + "learning_rate": 1.8373475362239628e-05, + "loss": 1.140932681664708e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1665, + "tokens/total": 25138668, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 538260 + }, + { + "epoch": 26.03125, + "grad_norm": 1.507762135588564e-05, + "learning_rate": 1.8331316976271658e-05, + "loss": 3.010793534485856e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1666, + "tokens/total": 25153744, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 538585 + }, + { + "epoch": 26.046875, + "grad_norm": 1.4052840924705379e-05, + "learning_rate": 1.828925415759822e-05, + "loss": 2.535284977511765e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1667, + "tokens/total": 25168668, + "tokens/train_per_sec_per_gpu": 9.67, + "tokens/trainable": 538911 + }, + { + "epoch": 26.0625, + "grad_norm": 2.6846972104976885e-05, + "learning_rate": 1.8247287015845e-05, + "loss": 3.706492464061739e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1668, + "tokens/total": 25183840, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 539233 + }, + { + "epoch": 26.078125, + "grad_norm": 9.907519597618375e-06, + "learning_rate": 1.8205415660388354e-05, + "loss": 1.9846407894874574e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1669, + "tokens/total": 25199016, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 539555 + }, + { + "epoch": 26.09375, + "grad_norm": 2.065640910586808e-05, + "learning_rate": 1.816364020035497e-05, + "loss": 3.09236213524855e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1670, + "tokens/total": 25214144, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 539875 + }, + { + "epoch": 26.109375, + "grad_norm": 1.7474489141022786e-05, + "learning_rate": 1.812196074462161e-05, + "loss": 3.672821549116634e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1671, + "tokens/total": 25229296, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 540198 + }, + { + "epoch": 26.125, + "grad_norm": 1.688633710728027e-05, + "learning_rate": 1.8080377401814823e-05, + "loss": 3.433904396388243e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1672, + "tokens/total": 25244320, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 540520 + }, + { + "epoch": 26.140625, + "grad_norm": 3.090012614848092e-05, + "learning_rate": 1.80388902803107e-05, + "loss": 6.234964189388847e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1673, + "tokens/total": 25259336, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 540843 + }, + { + "epoch": 26.15625, + "grad_norm": 1.7216889318660833e-05, + "learning_rate": 1.7997499488234513e-05, + "loss": 3.385138427347556e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1674, + "tokens/total": 25274464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 541167 + }, + { + "epoch": 26.171875, + "grad_norm": 9.563939784129616e-06, + "learning_rate": 1.795620513346049e-05, + "loss": 1.8823500624876033e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1675, + "tokens/total": 25289548, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 541491 + }, + { + "epoch": 26.1875, + "grad_norm": 1.4966720300435554e-05, + "learning_rate": 1.7915007323611526e-05, + "loss": 2.9833464054718206e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1676, + "tokens/total": 25304616, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 541809 + }, + { + "epoch": 26.203125, + "grad_norm": 1.918211273732595e-05, + "learning_rate": 1.7873906166058917e-05, + "loss": 4.830168336411589e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1677, + "tokens/total": 25319640, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 542129 + }, + { + "epoch": 26.21875, + "grad_norm": 7.293772341654403e-06, + "learning_rate": 1.783290176792201e-05, + "loss": 1.451990669920633e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1678, + "tokens/total": 25334800, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 542457 + }, + { + "epoch": 26.234375, + "grad_norm": 1.8006463506026193e-05, + "learning_rate": 1.779199423606802e-05, + "loss": 3.7464519664354157e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1679, + "tokens/total": 25350044, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 542787 + }, + { + "epoch": 26.25, + "grad_norm": 3.732422328539542e-06, + "learning_rate": 1.775118367711167e-05, + "loss": 6.978471134289066e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1680, + "tokens/total": 25365064, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 543108 + }, + { + "epoch": 26.265625, + "grad_norm": 1.1862853170896415e-05, + "learning_rate": 1.7710470197414998e-05, + "loss": 2.353327914761394e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1681, + "tokens/total": 25380168, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 543434 + }, + { + "epoch": 26.28125, + "grad_norm": 1.3841820873494726e-05, + "learning_rate": 1.766985390308697e-05, + "loss": 2.901199138705124e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1682, + "tokens/total": 25395220, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 543753 + }, + { + "epoch": 26.296875, + "grad_norm": 2.0526616935967468e-05, + "learning_rate": 1.7629334899983308e-05, + "loss": 2.6675741082726745e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1683, + "tokens/total": 25410384, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 544076 + }, + { + "epoch": 26.3125, + "grad_norm": 2.6315108698327094e-05, + "learning_rate": 1.7588913293706143e-05, + "loss": 3.2063982757790654e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1684, + "tokens/total": 25425476, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 544405 + }, + { + "epoch": 26.328125, + "grad_norm": 1.0301676411472727e-05, + "learning_rate": 1.754858918960378e-05, + "loss": 1.8806838397722458e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1685, + "tokens/total": 25440588, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 544730 + }, + { + "epoch": 26.34375, + "grad_norm": 1.136316768679535e-05, + "learning_rate": 1.7508362692770442e-05, + "loss": 2.509479202217335e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1686, + "tokens/total": 25455748, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 545054 + }, + { + "epoch": 26.359375, + "grad_norm": 1.1903000086022075e-05, + "learning_rate": 1.7468233908045872e-05, + "loss": 2.487178392129863e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1687, + "tokens/total": 25470876, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 545378 + }, + { + "epoch": 26.375, + "grad_norm": 1.4485181964118965e-05, + "learning_rate": 1.742820294001526e-05, + "loss": 2.3572818008688046e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1688, + "tokens/total": 25485920, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 545701 + }, + { + "epoch": 26.390625, + "grad_norm": 2.197578942286782e-05, + "learning_rate": 1.738826989300879e-05, + "loss": 4.350141580289346e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1689, + "tokens/total": 25501072, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 546027 + }, + { + "epoch": 26.40625, + "grad_norm": 6.842245966254268e-06, + "learning_rate": 1.7348434871101478e-05, + "loss": 1.3697882650376414e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1690, + "tokens/total": 25516172, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 546349 + }, + { + "epoch": 26.421875, + "grad_norm": 8.230919047491625e-05, + "learning_rate": 1.7308697978112828e-05, + "loss": 1.1349815167704946e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1691, + "tokens/total": 25531392, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 546670 + }, + { + "epoch": 26.4375, + "grad_norm": 2.3645632609259337e-05, + "learning_rate": 1.7269059317606643e-05, + "loss": 4.678511800193519e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1692, + "tokens/total": 25546420, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 546990 + }, + { + "epoch": 26.453125, + "grad_norm": 1.2751230315188877e-05, + "learning_rate": 1.7229518992890666e-05, + "loss": 1.851551019171893e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1693, + "tokens/total": 25561496, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 547314 + }, + { + "epoch": 26.46875, + "grad_norm": 1.4881487913953606e-05, + "learning_rate": 1.719007710701639e-05, + "loss": 2.7992669515697344e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1694, + "tokens/total": 25576600, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 547638 + }, + { + "epoch": 26.484375, + "grad_norm": 1.932913983182516e-05, + "learning_rate": 1.715073376277871e-05, + "loss": 3.9470788237849774e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1695, + "tokens/total": 25591720, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 547965 + }, + { + "epoch": 26.5, + "grad_norm": 5.992886872263625e-05, + "learning_rate": 1.711148906271574e-05, + "loss": 4.526692691797507e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1696, + "tokens/total": 25606872, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 548293 + }, + { + "epoch": 26.515625, + "grad_norm": 7.792485121171921e-06, + "learning_rate": 1.707234310910849e-05, + "loss": 1.5758644167362945e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1697, + "tokens/total": 25622016, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 548615 + }, + { + "epoch": 26.53125, + "grad_norm": 2.8183696485939436e-05, + "learning_rate": 1.7033296003980595e-05, + "loss": 5.168313919057255e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1698, + "tokens/total": 25637196, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 548939 + }, + { + "epoch": 26.546875, + "grad_norm": 1.433570469089318e-05, + "learning_rate": 1.6994347849098084e-05, + "loss": 2.4263619025077787e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1699, + "tokens/total": 25652400, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 549265 + }, + { + "epoch": 26.5625, + "grad_norm": 6.577192834811285e-06, + "learning_rate": 1.695549874596911e-05, + "loss": 1.0883124446081638e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1700, + "tokens/total": 25667476, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 549589 + }, + { + "epoch": 26.578125, + "grad_norm": 1.6574960682191886e-05, + "learning_rate": 1.6916748795843658e-05, + "loss": 2.6424299903737847e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1701, + "tokens/total": 25682560, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 549908 + }, + { + "epoch": 26.59375, + "grad_norm": 9.554019015922677e-06, + "learning_rate": 1.6878098099713293e-05, + "loss": 1.813072714185182e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1702, + "tokens/total": 25697584, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 550231 + }, + { + "epoch": 26.609375, + "grad_norm": 3.1168929126579314e-05, + "learning_rate": 1.6839546758310904e-05, + "loss": 5.633438604490948e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1703, + "tokens/total": 25712736, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 550559 + }, + { + "epoch": 26.625, + "grad_norm": 1.013230939861387e-05, + "learning_rate": 1.680109487211043e-05, + "loss": 1.9485629820792383e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1704, + "tokens/total": 25727804, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 550884 + }, + { + "epoch": 26.640625, + "grad_norm": 1.565935781400185e-05, + "learning_rate": 1.6762742541326647e-05, + "loss": 3.454455850260274e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1705, + "tokens/total": 25742828, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 551205 + }, + { + "epoch": 26.65625, + "grad_norm": 9.390014383825473e-06, + "learning_rate": 1.6724489865914805e-05, + "loss": 1.9172145471202384e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1706, + "tokens/total": 25757964, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 551527 + }, + { + "epoch": 26.671875, + "grad_norm": 7.101273240550654e-06, + "learning_rate": 1.668633694557048e-05, + "loss": 1.3164731171855237e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1707, + "tokens/total": 25772992, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 551852 + }, + { + "epoch": 26.6875, + "grad_norm": 1.803266604838427e-05, + "learning_rate": 1.6648283879729217e-05, + "loss": 3.790038363149506e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1708, + "tokens/total": 25787984, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 552179 + }, + { + "epoch": 26.703125, + "grad_norm": 2.5234514396288432e-05, + "learning_rate": 1.661033076756638e-05, + "loss": 3.0181621468727826e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1709, + "tokens/total": 25803048, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 552502 + }, + { + "epoch": 26.71875, + "grad_norm": 3.2212905352935195e-05, + "learning_rate": 1.6572477707996746e-05, + "loss": 3.083475235143851e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1710, + "tokens/total": 25818120, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 552826 + }, + { + "epoch": 26.734375, + "grad_norm": 2.8504489819169976e-05, + "learning_rate": 1.6534724799674412e-05, + "loss": 6.358001769513066e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1711, + "tokens/total": 25833148, + "tokens/train_per_sec_per_gpu": 8.92, + "tokens/trainable": 553149 + }, + { + "epoch": 26.75, + "grad_norm": 2.5703046048874967e-05, + "learning_rate": 1.6497072140992386e-05, + "loss": 3.815028435383283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1712, + "tokens/total": 25848284, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 553472 + }, + { + "epoch": 26.765625, + "grad_norm": 1.7028751244652085e-05, + "learning_rate": 1.6459519830082486e-05, + "loss": 3.342765069191955e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.01, + "memory/max_allocated (GiB)": 33.01, + "ppl": 1.0, + "step": 1713, + "tokens/total": 25863464, + "tokens/train_per_sec_per_gpu": 8.82, + "tokens/trainable": 553796 + }, + { + "epoch": 26.78125, + "grad_norm": 1.708783020148985e-05, + "learning_rate": 1.64220679648149e-05, + "loss": 3.2723795584388427e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1714, + "tokens/total": 25878620, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 554122 + }, + { + "epoch": 26.796875, + "grad_norm": 0.00011475582141429186, + "learning_rate": 1.6384716642798118e-05, + "loss": 1.0718524663388962e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1715, + "tokens/total": 25893648, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 554438 + }, + { + "epoch": 26.8125, + "grad_norm": 0.00013939087511971593, + "learning_rate": 1.6347465961378533e-05, + "loss": 1.4483779295915156e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1716, + "tokens/total": 25908836, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 554761 + }, + { + "epoch": 26.828125, + "grad_norm": 4.1579940443625674e-05, + "learning_rate": 1.6310316017640277e-05, + "loss": 2.31365220315638e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1717, + "tokens/total": 25923892, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 555084 + }, + { + "epoch": 26.84375, + "grad_norm": 3.3659322070889175e-05, + "learning_rate": 1.6273266908404898e-05, + "loss": 5.051117568655172e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1718, + "tokens/total": 25939004, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 555408 + }, + { + "epoch": 26.859375, + "grad_norm": 2.254985702165868e-05, + "learning_rate": 1.623631873023121e-05, + "loss": 3.1143167689151596e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1719, + "tokens/total": 25954084, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 555727 + }, + { + "epoch": 26.875, + "grad_norm": 1.149524177890271e-05, + "learning_rate": 1.6199471579414924e-05, + "loss": 2.559519032274693e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1720, + "tokens/total": 25969136, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 556050 + }, + { + "epoch": 26.890625, + "grad_norm": 1.0094465324073099e-05, + "learning_rate": 1.6162725551988453e-05, + "loss": 2.0524225874396507e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1721, + "tokens/total": 25984236, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 556372 + }, + { + "epoch": 26.90625, + "grad_norm": 3.464572728262283e-05, + "learning_rate": 1.612608074372068e-05, + "loss": 4.4761551976080227e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1722, + "tokens/total": 25999280, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 556695 + }, + { + "epoch": 26.921875, + "grad_norm": 1.8088368960889056e-05, + "learning_rate": 1.608953725011667e-05, + "loss": 2.682028252820601e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1723, + "tokens/total": 26014452, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 557016 + }, + { + "epoch": 26.9375, + "grad_norm": 1.5707471902715042e-05, + "learning_rate": 1.6053095166417448e-05, + "loss": 3.247565700803534e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1724, + "tokens/total": 26029416, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 557338 + }, + { + "epoch": 26.953125, + "grad_norm": 2.5381697923876345e-05, + "learning_rate": 1.6016754587599742e-05, + "loss": 5.066867743153125e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1725, + "tokens/total": 26044444, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 557661 + }, + { + "epoch": 26.96875, + "grad_norm": 1.754653021635022e-05, + "learning_rate": 1.5980515608375717e-05, + "loss": 3.7541434494414716e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1726, + "tokens/total": 26059536, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 557987 + }, + { + "epoch": 26.984375, + "grad_norm": 6.272376140259439e-06, + "learning_rate": 1.594437832319276e-05, + "loss": 1.1869548899312576e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1727, + "tokens/total": 26074684, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 558309 + }, + { + "epoch": 27.0, + "grad_norm": 1.1199465916433837e-05, + "learning_rate": 1.590834282623324e-05, + "loss": 2.039530073716378e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1728, + "tokens/total": 26089776, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 558630 + }, + { + "epoch": 27.015625, + "grad_norm": 1.5308742149500176e-05, + "learning_rate": 1.587240921141418e-05, + "loss": 2.5065600084417383e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1729, + "tokens/total": 26104836, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 558952 + }, + { + "epoch": 27.03125, + "grad_norm": 6.34040916338563e-05, + "learning_rate": 1.5836577572387136e-05, + "loss": 3.0047655741327617e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1730, + "tokens/total": 26119808, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 559276 + }, + { + "epoch": 27.046875, + "grad_norm": 1.6484083971590735e-05, + "learning_rate": 1.5800848002537854e-05, + "loss": 2.48086394094571e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1731, + "tokens/total": 26135008, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 559602 + }, + { + "epoch": 27.0625, + "grad_norm": 6.733540885761613e-06, + "learning_rate": 1.5765220594986108e-05, + "loss": 1.5204064141016715e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1732, + "tokens/total": 26150168, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 559928 + }, + { + "epoch": 27.078125, + "grad_norm": 8.736526069696993e-06, + "learning_rate": 1.572969544258534e-05, + "loss": 1.638023547911871e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1733, + "tokens/total": 26165200, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 560253 + }, + { + "epoch": 27.09375, + "grad_norm": 1.4880755770718679e-05, + "learning_rate": 1.5694272637922552e-05, + "loss": 3.0043491960896063e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1734, + "tokens/total": 26180276, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 560573 + }, + { + "epoch": 27.109375, + "grad_norm": 9.321833204012364e-06, + "learning_rate": 1.565895227331799e-05, + "loss": 1.9800485517862398e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1735, + "tokens/total": 26195316, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 560901 + }, + { + "epoch": 27.125, + "grad_norm": 1.106218587665353e-05, + "learning_rate": 1.5623734440824894e-05, + "loss": 2.3197257803531102e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1736, + "tokens/total": 26210372, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 561229 + }, + { + "epoch": 27.140625, + "grad_norm": 3.121475674561225e-05, + "learning_rate": 1.5588619232229302e-05, + "loss": 5.008970447306638e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1737, + "tokens/total": 26225508, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 561553 + }, + { + "epoch": 27.15625, + "grad_norm": 1.725515721773263e-05, + "learning_rate": 1.5553606739049802e-05, + "loss": 3.1657492627346073e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1738, + "tokens/total": 26240604, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 561875 + }, + { + "epoch": 27.171875, + "grad_norm": 2.0479226805036888e-05, + "learning_rate": 1.5518697052537252e-05, + "loss": 4.328777265527606e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1739, + "tokens/total": 26255788, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 562200 + }, + { + "epoch": 27.1875, + "grad_norm": 1.0336267223465256e-05, + "learning_rate": 1.548389026367458e-05, + "loss": 2.168342092545572e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1740, + "tokens/total": 26270832, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 562521 + }, + { + "epoch": 27.203125, + "grad_norm": 3.472311072982848e-05, + "learning_rate": 1.5449186463176544e-05, + "loss": 6.696618584101088e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1741, + "tokens/total": 26285940, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 562839 + }, + { + "epoch": 27.21875, + "grad_norm": 9.998497262131423e-06, + "learning_rate": 1.5414585741489483e-05, + "loss": 1.9818475038846373e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1742, + "tokens/total": 26301072, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 563163 + }, + { + "epoch": 27.234375, + "grad_norm": 8.031962352106348e-06, + "learning_rate": 1.5380088188791114e-05, + "loss": 1.5150315846312878e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1743, + "tokens/total": 26316136, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 563484 + }, + { + "epoch": 27.25, + "grad_norm": 1.9238019376643933e-05, + "learning_rate": 1.5345693894990237e-05, + "loss": 4.4059774495508464e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1744, + "tokens/total": 26331332, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 563808 + }, + { + "epoch": 27.265625, + "grad_norm": 6.256763299461454e-06, + "learning_rate": 1.531140294972655e-05, + "loss": 1.2189995857170288e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1745, + "tokens/total": 26346464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 564129 + }, + { + "epoch": 27.28125, + "grad_norm": 2.0716122890007682e-05, + "learning_rate": 1.5277215442370385e-05, + "loss": 4.483669613364327e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1746, + "tokens/total": 26361668, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 564452 + }, + { + "epoch": 27.296875, + "grad_norm": 2.1896104954066686e-05, + "learning_rate": 1.524313146202252e-05, + "loss": 3.734713800440659e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1747, + "tokens/total": 26376888, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 564775 + }, + { + "epoch": 27.3125, + "grad_norm": 8.62186607264448e-06, + "learning_rate": 1.5209151097513898e-05, + "loss": 1.7523382211948046e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1748, + "tokens/total": 26391924, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 565093 + }, + { + "epoch": 27.328125, + "grad_norm": 2.3566348318126984e-05, + "learning_rate": 1.5175274437405423e-05, + "loss": 3.3366134744028386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1749, + "tokens/total": 26407012, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 565415 + }, + { + "epoch": 27.34375, + "grad_norm": 1.8717664715950377e-05, + "learning_rate": 1.5141501569987693e-05, + "loss": 2.977969870698871e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1750, + "tokens/total": 26422096, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 565734 + }, + { + "epoch": 27.359375, + "grad_norm": 8.070656622294337e-05, + "learning_rate": 1.510783258328084e-05, + "loss": 1.2917469121020986e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1751, + "tokens/total": 26437164, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 566061 + }, + { + "epoch": 27.375, + "grad_norm": 1.7950362234842032e-05, + "learning_rate": 1.5074267565034234e-05, + "loss": 3.2098984092954197e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1752, + "tokens/total": 26452252, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 566386 + }, + { + "epoch": 27.390625, + "grad_norm": 1.3401237993093673e-05, + "learning_rate": 1.5040806602726277e-05, + "loss": 2.1016276718910376e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1753, + "tokens/total": 26467400, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 566710 + }, + { + "epoch": 27.40625, + "grad_norm": 8.833386345941108e-06, + "learning_rate": 1.5007449783564184e-05, + "loss": 1.699826128742643e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1754, + "tokens/total": 26482648, + "tokens/train_per_sec_per_gpu": 8.79, + "tokens/trainable": 567034 + }, + { + "epoch": 27.421875, + "grad_norm": 9.98876930680126e-06, + "learning_rate": 1.4974197194483742e-05, + "loss": 2.114290822419207e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1755, + "tokens/total": 26497772, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 567362 + }, + { + "epoch": 27.4375, + "grad_norm": 4.575460479827598e-05, + "learning_rate": 1.4941048922149097e-05, + "loss": 7.329226150432078e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1756, + "tokens/total": 26512856, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 567682 + }, + { + "epoch": 27.453125, + "grad_norm": 1.243073620571522e-05, + "learning_rate": 1.4908005052952528e-05, + "loss": 2.1065866917524545e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1757, + "tokens/total": 26527988, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 568004 + }, + { + "epoch": 27.46875, + "grad_norm": 1.0237574315397069e-05, + "learning_rate": 1.487506567301419e-05, + "loss": 2.1193943666730775e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1758, + "tokens/total": 26543248, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 568330 + }, + { + "epoch": 27.484375, + "grad_norm": 6.066151854611235e-06, + "learning_rate": 1.4842230868181925e-05, + "loss": 1.2197726562135358e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1759, + "tokens/total": 26558268, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 568653 + }, + { + "epoch": 27.5, + "grad_norm": 2.2678284949506633e-05, + "learning_rate": 1.4809500724031056e-05, + "loss": 4.837976348426309e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1760, + "tokens/total": 26573296, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 568976 + }, + { + "epoch": 27.515625, + "grad_norm": 2.3314029022003524e-05, + "learning_rate": 1.477687532586407e-05, + "loss": 3.9413075114680396e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1761, + "tokens/total": 26588388, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 569300 + }, + { + "epoch": 27.53125, + "grad_norm": 0.00016100230277515948, + "learning_rate": 1.4744354758710538e-05, + "loss": 1.417173734807875e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1762, + "tokens/total": 26603384, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 569626 + }, + { + "epoch": 27.546875, + "grad_norm": 1.2848717233282514e-05, + "learning_rate": 1.4711939107326761e-05, + "loss": 2.4356506855838234e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1763, + "tokens/total": 26618504, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 569953 + }, + { + "epoch": 27.5625, + "grad_norm": 8.349126801476814e-06, + "learning_rate": 1.4679628456195632e-05, + "loss": 1.7754004488779174e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1764, + "tokens/total": 26633596, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 570275 + }, + { + "epoch": 27.578125, + "grad_norm": 1.8816332158166915e-05, + "learning_rate": 1.464742288952636e-05, + "loss": 3.122359260032681e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1765, + "tokens/total": 26648724, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 570599 + }, + { + "epoch": 27.59375, + "grad_norm": 1.0163017577724531e-05, + "learning_rate": 1.4615322491254324e-05, + "loss": 1.805190379400301e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1766, + "tokens/total": 26663768, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 570922 + }, + { + "epoch": 27.609375, + "grad_norm": 1.4959176041884348e-05, + "learning_rate": 1.4583327345040779e-05, + "loss": 2.9402173140624654e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1767, + "tokens/total": 26678732, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 571243 + }, + { + "epoch": 27.625, + "grad_norm": 1.9049735783482902e-05, + "learning_rate": 1.4551437534272663e-05, + "loss": 2.935134375547932e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1768, + "tokens/total": 26693984, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 571565 + }, + { + "epoch": 27.640625, + "grad_norm": 2.597888305899687e-05, + "learning_rate": 1.4519653142062385e-05, + "loss": 1.8902197496117878e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1769, + "tokens/total": 26709048, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 571888 + }, + { + "epoch": 27.65625, + "grad_norm": 2.8063026547897607e-05, + "learning_rate": 1.4487974251247646e-05, + "loss": 4.3432720531200175e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1770, + "tokens/total": 26724236, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 572212 + }, + { + "epoch": 27.671875, + "grad_norm": 1.4272887710831128e-05, + "learning_rate": 1.4456400944391146e-05, + "loss": 2.9155049219298235e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1771, + "tokens/total": 26739400, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 572535 + }, + { + "epoch": 27.6875, + "grad_norm": 8.008511031221133e-06, + "learning_rate": 1.4424933303780411e-05, + "loss": 1.5642680750715954e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1772, + "tokens/total": 26754612, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 572852 + }, + { + "epoch": 27.703125, + "grad_norm": 1.770760900399182e-05, + "learning_rate": 1.439357141142759e-05, + "loss": 3.5766245787272055e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1773, + "tokens/total": 26769748, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 573179 + }, + { + "epoch": 27.71875, + "grad_norm": 9.19958074518945e-06, + "learning_rate": 1.4362315349069214e-05, + "loss": 2.0704847258912196e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1774, + "tokens/total": 26784856, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 573501 + }, + { + "epoch": 27.734375, + "grad_norm": 9.346610568172764e-06, + "learning_rate": 1.4331165198166013e-05, + "loss": 2.001538916829304e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1775, + "tokens/total": 26799916, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 573825 + }, + { + "epoch": 27.75, + "grad_norm": 3.0114308174233884e-05, + "learning_rate": 1.4300121039902682e-05, + "loss": 5.897442747482273e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1776, + "tokens/total": 26815020, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 574145 + }, + { + "epoch": 27.765625, + "grad_norm": 8.97269001143286e-06, + "learning_rate": 1.4269182955187651e-05, + "loss": 1.9218120428377006e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1777, + "tokens/total": 26830080, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 574467 + }, + { + "epoch": 27.78125, + "grad_norm": 8.77154161571525e-06, + "learning_rate": 1.4238351024652918e-05, + "loss": 1.8208586993750941e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1778, + "tokens/total": 26845148, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 574791 + }, + { + "epoch": 27.796875, + "grad_norm": 1.3865632354281843e-05, + "learning_rate": 1.4207625328653836e-05, + "loss": 2.494058435331681e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1779, + "tokens/total": 26860380, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 575113 + }, + { + "epoch": 27.8125, + "grad_norm": 1.006775983114494e-05, + "learning_rate": 1.4177005947268834e-05, + "loss": 2.0631611619137402e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1780, + "tokens/total": 26875444, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 575433 + }, + { + "epoch": 27.828125, + "grad_norm": 7.545062544522807e-06, + "learning_rate": 1.414649296029931e-05, + "loss": 1.498484749618001e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1781, + "tokens/total": 26890548, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 575758 + }, + { + "epoch": 27.84375, + "grad_norm": 2.7258571208221838e-05, + "learning_rate": 1.4116086447269341e-05, + "loss": 3.158816355153249e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1782, + "tokens/total": 26905592, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 576081 + }, + { + "epoch": 27.859375, + "grad_norm": 3.0004832296981476e-05, + "learning_rate": 1.4085786487425545e-05, + "loss": 4.865910341322888e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1783, + "tokens/total": 26920684, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 576408 + }, + { + "epoch": 27.875, + "grad_norm": 5.240008249529637e-06, + "learning_rate": 1.405559315973677e-05, + "loss": 9.665070166420264e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1784, + "tokens/total": 26935808, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 576732 + }, + { + "epoch": 27.890625, + "grad_norm": 1.5380093827843666e-05, + "learning_rate": 1.4025506542894032e-05, + "loss": 3.3570239565960946e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1785, + "tokens/total": 26950860, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 577056 + }, + { + "epoch": 27.90625, + "grad_norm": 1.4769061635888647e-05, + "learning_rate": 1.3995526715310176e-05, + "loss": 2.822827411819162e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1786, + "tokens/total": 26965972, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 577383 + }, + { + "epoch": 27.921875, + "grad_norm": 7.029739208519459e-06, + "learning_rate": 1.3965653755119778e-05, + "loss": 1.4682642301977467e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1787, + "tokens/total": 26981100, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 577708 + }, + { + "epoch": 27.9375, + "grad_norm": 2.410988963674754e-05, + "learning_rate": 1.3935887740178832e-05, + "loss": 4.351698805749038e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1788, + "tokens/total": 26996160, + "tokens/train_per_sec_per_gpu": 8.87, + "tokens/trainable": 578030 + }, + { + "epoch": 27.953125, + "grad_norm": 0.00011136336252093315, + "learning_rate": 1.3906228748064665e-05, + "loss": 1.3107342056173366e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1789, + "tokens/total": 27011240, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 578350 + }, + { + "epoch": 27.96875, + "grad_norm": 5.284467260935344e-05, + "learning_rate": 1.3876676856075644e-05, + "loss": 7.260025540745119e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1790, + "tokens/total": 27026260, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 578673 + }, + { + "epoch": 27.984375, + "grad_norm": 3.170741183566861e-05, + "learning_rate": 1.3847232141231017e-05, + "loss": 5.156756515134475e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1791, + "tokens/total": 27041364, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 578995 + }, + { + "epoch": 28.0, + "grad_norm": 1.127932409872301e-05, + "learning_rate": 1.381789468027069e-05, + "loss": 2.5181705609611527e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1792, + "tokens/total": 27056476, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 579320 + }, + { + "epoch": 28.015625, + "grad_norm": 2.8710635888273828e-05, + "learning_rate": 1.3788664549655077e-05, + "loss": 5.876510158486781e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1793, + "tokens/total": 27071592, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 579645 + }, + { + "epoch": 28.03125, + "grad_norm": 9.895234688883647e-05, + "learning_rate": 1.3759541825564826e-05, + "loss": 8.725961606614874e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1794, + "tokens/total": 27086620, + "tokens/train_per_sec_per_gpu": 10.02, + "tokens/trainable": 579970 + }, + { + "epoch": 28.046875, + "grad_norm": 5.798015263280831e-05, + "learning_rate": 1.3730526583900671e-05, + "loss": 8.698302735865582e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1795, + "tokens/total": 27101828, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 580294 + }, + { + "epoch": 28.0625, + "grad_norm": 1.8622051356942393e-05, + "learning_rate": 1.3701618900283219e-05, + "loss": 2.9004434054513695e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1796, + "tokens/total": 27116992, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 580618 + }, + { + "epoch": 28.078125, + "grad_norm": 1.6327088815160096e-05, + "learning_rate": 1.3672818850052752e-05, + "loss": 2.3000774262982304e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1797, + "tokens/total": 27132184, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 580946 + }, + { + "epoch": 28.09375, + "grad_norm": 1.0781125638459343e-05, + "learning_rate": 1.3644126508269059e-05, + "loss": 2.0816116830246756e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1798, + "tokens/total": 27147260, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 581273 + }, + { + "epoch": 28.109375, + "grad_norm": 2.1183366698096506e-05, + "learning_rate": 1.3615541949711171e-05, + "loss": 3.959080459026154e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1799, + "tokens/total": 27162400, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 581595 + }, + { + "epoch": 28.125, + "grad_norm": 1.9014221834368072e-05, + "learning_rate": 1.3587065248877252e-05, + "loss": 4.0846722981768835e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1800, + "tokens/total": 27177428, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 581914 + }, + { + "epoch": 28.140625, + "grad_norm": 3.437084160395898e-05, + "learning_rate": 1.3558696479984334e-05, + "loss": 6.974984785301785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1801, + "tokens/total": 27192596, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 582236 + }, + { + "epoch": 28.15625, + "grad_norm": 1.3638349628308788e-05, + "learning_rate": 1.3530435716968188e-05, + "loss": 2.5111538093369745e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1802, + "tokens/total": 27207772, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 582556 + }, + { + "epoch": 28.171875, + "grad_norm": 1.756194615154527e-05, + "learning_rate": 1.3502283033483032e-05, + "loss": 3.150006193664012e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1803, + "tokens/total": 27222944, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 582880 + }, + { + "epoch": 28.1875, + "grad_norm": 2.2211039322428405e-05, + "learning_rate": 1.3474238502901482e-05, + "loss": 4.7422938109775714e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1804, + "tokens/total": 27238096, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 583202 + }, + { + "epoch": 28.203125, + "grad_norm": 2.0460905943764374e-05, + "learning_rate": 1.3446302198314217e-05, + "loss": 4.05049831897486e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1805, + "tokens/total": 27253088, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 583518 + }, + { + "epoch": 28.21875, + "grad_norm": 1.1221286513318773e-05, + "learning_rate": 1.3418474192529914e-05, + "loss": 2.0240916853708768e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1806, + "tokens/total": 27268100, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 583846 + }, + { + "epoch": 28.234375, + "grad_norm": 1.1699700735334773e-05, + "learning_rate": 1.339075455807492e-05, + "loss": 2.513777417334495e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1807, + "tokens/total": 27283176, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 584165 + }, + { + "epoch": 28.25, + "grad_norm": 1.2705786502920091e-05, + "learning_rate": 1.3363143367193213e-05, + "loss": 2.1751364442934573e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1808, + "tokens/total": 27298308, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 584484 + }, + { + "epoch": 28.265625, + "grad_norm": 3.379157715244219e-05, + "learning_rate": 1.3335640691846104e-05, + "loss": 2.94440553716413e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1809, + "tokens/total": 27313564, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 584808 + }, + { + "epoch": 28.28125, + "grad_norm": 1.313696975557832e-05, + "learning_rate": 1.3308246603712101e-05, + "loss": 2.901714424297097e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1810, + "tokens/total": 27328724, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 585131 + }, + { + "epoch": 28.296875, + "grad_norm": 1.2426772627804894e-05, + "learning_rate": 1.3280961174186685e-05, + "loss": 1.963152129746959e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1811, + "tokens/total": 27343736, + "tokens/train_per_sec_per_gpu": 10.07, + "tokens/trainable": 585456 + }, + { + "epoch": 28.3125, + "grad_norm": 1.0397198821010534e-05, + "learning_rate": 1.3253784474382182e-05, + "loss": 2.1323971566289401e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1812, + "tokens/total": 27358912, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 585785 + }, + { + "epoch": 28.328125, + "grad_norm": 1.5642424841644242e-05, + "learning_rate": 1.3226716575127523e-05, + "loss": 2.8898756454509567e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1813, + "tokens/total": 27373972, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 586112 + }, + { + "epoch": 28.34375, + "grad_norm": 1.276227612834191e-05, + "learning_rate": 1.3199757546968072e-05, + "loss": 2.909919203375466e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1814, + "tokens/total": 27389104, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 586436 + }, + { + "epoch": 28.359375, + "grad_norm": 1.1631118468358181e-05, + "learning_rate": 1.317290746016546e-05, + "loss": 2.105115868289431e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1815, + "tokens/total": 27404256, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 586760 + }, + { + "epoch": 28.375, + "grad_norm": 2.827637035807129e-05, + "learning_rate": 1.3146166384697389e-05, + "loss": 3.8131292967591435e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1816, + "tokens/total": 27419320, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 587081 + }, + { + "epoch": 28.390625, + "grad_norm": 8.35687114886241e-06, + "learning_rate": 1.311953439025745e-05, + "loss": 1.6183867046493106e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1817, + "tokens/total": 27434472, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 587403 + }, + { + "epoch": 28.40625, + "grad_norm": 1.0089405805047136e-05, + "learning_rate": 1.3093011546254953e-05, + "loss": 2.0367072295357502e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1818, + "tokens/total": 27449492, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 587723 + }, + { + "epoch": 28.421875, + "grad_norm": 7.49484242987819e-05, + "learning_rate": 1.306659792181472e-05, + "loss": 1.0364508398197358e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1819, + "tokens/total": 27464616, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 588045 + }, + { + "epoch": 28.4375, + "grad_norm": 1.0778055184346158e-05, + "learning_rate": 1.3040293585776916e-05, + "loss": 2.1211107537055796e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1820, + "tokens/total": 27479816, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 588371 + }, + { + "epoch": 28.453125, + "grad_norm": 2.7706506443792023e-05, + "learning_rate": 1.3014098606696909e-05, + "loss": 3.5298302236697054e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1821, + "tokens/total": 27495072, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 588694 + }, + { + "epoch": 28.46875, + "grad_norm": 1.2086034985259175e-05, + "learning_rate": 1.2988013052845022e-05, + "loss": 2.0647325982281473e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1822, + "tokens/total": 27510196, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 589016 + }, + { + "epoch": 28.484375, + "grad_norm": 1.282368157262681e-05, + "learning_rate": 1.2962036992206406e-05, + "loss": 1.887266023459233e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1823, + "tokens/total": 27525296, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 589335 + }, + { + "epoch": 28.5, + "grad_norm": 6.218464477569796e-06, + "learning_rate": 1.2936170492480828e-05, + "loss": 1.2749219990837446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1824, + "tokens/total": 27540288, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 589661 + }, + { + "epoch": 28.515625, + "grad_norm": 1.828308631957043e-05, + "learning_rate": 1.2910413621082551e-05, + "loss": 3.9277060182030255e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1825, + "tokens/total": 27555456, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 589986 + }, + { + "epoch": 28.53125, + "grad_norm": 7.52272853787872e-06, + "learning_rate": 1.2884766445140084e-05, + "loss": 1.4097921052780293e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1826, + "tokens/total": 27570532, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 590311 + }, + { + "epoch": 28.546875, + "grad_norm": 8.019937013159506e-06, + "learning_rate": 1.2859229031496064e-05, + "loss": 1.6753909903854947e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1827, + "tokens/total": 27585548, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 590634 + }, + { + "epoch": 28.5625, + "grad_norm": 1.8432008801028132e-05, + "learning_rate": 1.2833801446707045e-05, + "loss": 2.703113750612829e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1828, + "tokens/total": 27600660, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 590960 + }, + { + "epoch": 28.578125, + "grad_norm": 1.079344110621605e-05, + "learning_rate": 1.2808483757043355e-05, + "loss": 2.1949387019049027e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1829, + "tokens/total": 27615664, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 591285 + }, + { + "epoch": 28.59375, + "grad_norm": 1.0355586709920317e-05, + "learning_rate": 1.2783276028488907e-05, + "loss": 2.0789761379091942e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1830, + "tokens/total": 27630736, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 591606 + }, + { + "epoch": 28.609375, + "grad_norm": 4.236560926074162e-06, + "learning_rate": 1.2758178326741027e-05, + "loss": 7.725177653128412e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1831, + "tokens/total": 27645796, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 591931 + }, + { + "epoch": 28.625, + "grad_norm": 1.1969351362495217e-05, + "learning_rate": 1.2733190717210284e-05, + "loss": 2.4853335389707354e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1832, + "tokens/total": 27660764, + "tokens/train_per_sec_per_gpu": 8.92, + "tokens/trainable": 592251 + }, + { + "epoch": 28.640625, + "grad_norm": 3.1693733035353944e-05, + "learning_rate": 1.270831326502032e-05, + "loss": 5.709023298550164e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1833, + "tokens/total": 27675848, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 592571 + }, + { + "epoch": 28.65625, + "grad_norm": 6.424582352337893e-06, + "learning_rate": 1.2683546035007688e-05, + "loss": 9.682638335561933e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1834, + "tokens/total": 27690904, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 592896 + }, + { + "epoch": 28.671875, + "grad_norm": 3.86581668863073e-05, + "learning_rate": 1.2658889091721654e-05, + "loss": 6.18281262632081e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1835, + "tokens/total": 27705948, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 593221 + }, + { + "epoch": 28.6875, + "grad_norm": 8.301987691083923e-06, + "learning_rate": 1.2634342499424082e-05, + "loss": 1.6370097455364885e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1836, + "tokens/total": 27721120, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 593548 + }, + { + "epoch": 28.703125, + "grad_norm": 5.3625175496563315e-05, + "learning_rate": 1.260990632208922e-05, + "loss": 7.014593847998185e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1837, + "tokens/total": 27736312, + "tokens/train_per_sec_per_gpu": 8.85, + "tokens/trainable": 593872 + }, + { + "epoch": 28.71875, + "grad_norm": 1.2745299500238616e-05, + "learning_rate": 1.2585580623403544e-05, + "loss": 2.133188701236577e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1838, + "tokens/total": 27751464, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 594196 + }, + { + "epoch": 28.734375, + "grad_norm": 2.077351018670015e-05, + "learning_rate": 1.2561365466765592e-05, + "loss": 4.0698958514440164e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1839, + "tokens/total": 27766576, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 594516 + }, + { + "epoch": 28.75, + "grad_norm": 1.294104276894359e-05, + "learning_rate": 1.253726091528583e-05, + "loss": 2.582949889529118e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1840, + "tokens/total": 27781724, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 594840 + }, + { + "epoch": 28.765625, + "grad_norm": 4.652914867619984e-05, + "learning_rate": 1.251326703178643e-05, + "loss": 2.0522682575574436e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1841, + "tokens/total": 27796844, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 595166 + }, + { + "epoch": 28.78125, + "grad_norm": 3.008591193065513e-05, + "learning_rate": 1.2489383878801151e-05, + "loss": 4.5823202299288823e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1842, + "tokens/total": 27811944, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 595490 + }, + { + "epoch": 28.796875, + "grad_norm": 1.0593084880383685e-05, + "learning_rate": 1.246561151857516e-05, + "loss": 2.2231043317333388e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1843, + "tokens/total": 27826968, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 595812 + }, + { + "epoch": 28.8125, + "grad_norm": 1.9429338863119483e-05, + "learning_rate": 1.244195001306488e-05, + "loss": 1.8836196602478594e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1844, + "tokens/total": 27842012, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 596134 + }, + { + "epoch": 28.828125, + "grad_norm": 1.1641817764029838e-05, + "learning_rate": 1.2418399423937805e-05, + "loss": 1.1423554013845205e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1845, + "tokens/total": 27857132, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 596456 + }, + { + "epoch": 28.84375, + "grad_norm": 1.5534040358033963e-05, + "learning_rate": 1.2394959812572368e-05, + "loss": 2.5822015459198155e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1846, + "tokens/total": 27872212, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 596783 + }, + { + "epoch": 28.859375, + "grad_norm": 1.1093743523815647e-05, + "learning_rate": 1.2371631240057752e-05, + "loss": 1.7564653376211936e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1847, + "tokens/total": 27887348, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 597107 + }, + { + "epoch": 28.875, + "grad_norm": 9.979840797313955e-06, + "learning_rate": 1.2348413767193759e-05, + "loss": 1.8956168901240744e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1848, + "tokens/total": 27902384, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 597428 + }, + { + "epoch": 28.890625, + "grad_norm": 0.0001451041316613555, + "learning_rate": 1.2325307454490642e-05, + "loss": 1.2540815532702254e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1849, + "tokens/total": 27917500, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 597755 + }, + { + "epoch": 28.90625, + "grad_norm": 1.3303013474796899e-05, + "learning_rate": 1.2302312362168934e-05, + "loss": 2.882856620090024e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1850, + "tokens/total": 27932480, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 598073 + }, + { + "epoch": 28.921875, + "grad_norm": 1.089604393200716e-05, + "learning_rate": 1.2279428550159312e-05, + "loss": 1.9176783894181426e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1851, + "tokens/total": 27947692, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 598398 + }, + { + "epoch": 28.9375, + "grad_norm": 3.7186524423304945e-05, + "learning_rate": 1.2256656078102408e-05, + "loss": 4.698161433225323e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1852, + "tokens/total": 27962644, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 598717 + }, + { + "epoch": 28.953125, + "grad_norm": 1.0314109204045963e-05, + "learning_rate": 1.2233995005348711e-05, + "loss": 1.4683254789815692e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1853, + "tokens/total": 27977644, + "tokens/train_per_sec_per_gpu": 9.85, + "tokens/trainable": 599036 + }, + { + "epoch": 28.96875, + "grad_norm": 1.1664391422527842e-05, + "learning_rate": 1.2211445390958343e-05, + "loss": 2.3535750415248913e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1854, + "tokens/total": 27992784, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 599360 + }, + { + "epoch": 28.984375, + "grad_norm": 2.2983202143223025e-05, + "learning_rate": 1.2189007293700957e-05, + "loss": 3.934616756851028e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1855, + "tokens/total": 28007904, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 599686 + }, + { + "epoch": 29.0, + "grad_norm": 3.208417911082506e-05, + "learning_rate": 1.2166680772055556e-05, + "loss": 2.764373334684933e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1856, + "tokens/total": 28022940, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 600010 + }, + { + "epoch": 29.015625, + "grad_norm": 8.055651960603427e-06, + "learning_rate": 1.2144465884210378e-05, + "loss": 1.1116791398535497e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1857, + "tokens/total": 28038132, + "tokens/train_per_sec_per_gpu": 8.9, + "tokens/trainable": 600334 + }, + { + "epoch": 29.03125, + "grad_norm": 1.7299964383710176e-05, + "learning_rate": 1.2122362688062671e-05, + "loss": 3.1467544658880797e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1858, + "tokens/total": 28053132, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 600655 + }, + { + "epoch": 29.046875, + "grad_norm": 0.00013976563059259206, + "learning_rate": 1.2100371241218626e-05, + "loss": 1.4031820683158003e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1859, + "tokens/total": 28068220, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 600979 + }, + { + "epoch": 29.0625, + "grad_norm": 1.1101731615781318e-05, + "learning_rate": 1.207849160099317e-05, + "loss": 2.3388062686535704e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1860, + "tokens/total": 28083256, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 601302 + }, + { + "epoch": 29.078125, + "grad_norm": 2.045767178060487e-05, + "learning_rate": 1.2056723824409846e-05, + "loss": 3.8043756944716733e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1861, + "tokens/total": 28098408, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 601629 + }, + { + "epoch": 29.09375, + "grad_norm": 1.338845231657615e-05, + "learning_rate": 1.2035067968200637e-05, + "loss": 2.63431331859465e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1862, + "tokens/total": 28113512, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 601949 + }, + { + "epoch": 29.109375, + "grad_norm": 2.6296922442270443e-05, + "learning_rate": 1.2013524088805858e-05, + "loss": 5.399256792770757e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1863, + "tokens/total": 28128740, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 602271 + }, + { + "epoch": 29.125, + "grad_norm": 9.836249773798045e-06, + "learning_rate": 1.199209224237397e-05, + "loss": 2.016146964933796e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1864, + "tokens/total": 28143808, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 602596 + }, + { + "epoch": 29.140625, + "grad_norm": 2.0627150661312044e-05, + "learning_rate": 1.197077248476145e-05, + "loss": 2.404783003839839e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1865, + "tokens/total": 28158936, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 602919 + }, + { + "epoch": 29.15625, + "grad_norm": 2.537706313887611e-05, + "learning_rate": 1.1949564871532653e-05, + "loss": 3.8546457403754175e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1866, + "tokens/total": 28174140, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 603249 + }, + { + "epoch": 29.171875, + "grad_norm": 6.854383536847308e-05, + "learning_rate": 1.1928469457959648e-05, + "loss": 9.390921036356303e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1867, + "tokens/total": 28189212, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 603572 + }, + { + "epoch": 29.1875, + "grad_norm": 1.0834958629857283e-05, + "learning_rate": 1.1907486299022106e-05, + "loss": 2.1075236134038278e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1868, + "tokens/total": 28204312, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 603898 + }, + { + "epoch": 29.203125, + "grad_norm": 2.605345616757404e-05, + "learning_rate": 1.1886615449407105e-05, + "loss": 4.997341989110282e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1869, + "tokens/total": 28219420, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 604223 + }, + { + "epoch": 29.21875, + "grad_norm": 1.293555396841839e-05, + "learning_rate": 1.1865856963509043e-05, + "loss": 2.73707627229669e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1870, + "tokens/total": 28234496, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 604548 + }, + { + "epoch": 29.234375, + "grad_norm": 2.678318378457334e-05, + "learning_rate": 1.1845210895429463e-05, + "loss": 3.2632306101731956e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1871, + "tokens/total": 28249660, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 604871 + }, + { + "epoch": 29.25, + "grad_norm": 1.652997343626339e-05, + "learning_rate": 1.1824677298976924e-05, + "loss": 2.1980505948704376e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1872, + "tokens/total": 28264744, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 605202 + }, + { + "epoch": 29.265625, + "grad_norm": 2.0757082893396728e-05, + "learning_rate": 1.1804256227666843e-05, + "loss": 4.0945838009065483e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1873, + "tokens/total": 28279748, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 605528 + }, + { + "epoch": 29.28125, + "grad_norm": 8.131177310133353e-06, + "learning_rate": 1.1783947734721387e-05, + "loss": 1.576239725409323e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1874, + "tokens/total": 28294824, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 605846 + }, + { + "epoch": 29.296875, + "grad_norm": 2.0063534975633956e-05, + "learning_rate": 1.1763751873069309e-05, + "loss": 4.427661508543679e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1875, + "tokens/total": 28309928, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 606169 + }, + { + "epoch": 29.3125, + "grad_norm": 8.534272637916729e-06, + "learning_rate": 1.174366869534583e-05, + "loss": 1.598627363819105e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1876, + "tokens/total": 28325036, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 606493 + }, + { + "epoch": 29.328125, + "grad_norm": 2.460130417603068e-05, + "learning_rate": 1.172369825389246e-05, + "loss": 4.4087352080168785e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1877, + "tokens/total": 28340156, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 606812 + }, + { + "epoch": 29.34375, + "grad_norm": 1.7705246136756614e-05, + "learning_rate": 1.1703840600756928e-05, + "loss": 1.5181453250079358e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1878, + "tokens/total": 28355312, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 607131 + }, + { + "epoch": 29.359375, + "grad_norm": 1.4587425539502874e-05, + "learning_rate": 1.1684095787692985e-05, + "loss": 3.0665034955745796e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1879, + "tokens/total": 28370432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 607452 + }, + { + "epoch": 29.375, + "grad_norm": 6.322739864117466e-06, + "learning_rate": 1.1664463866160308e-05, + "loss": 1.215411913335629e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1880, + "tokens/total": 28385528, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 607773 + }, + { + "epoch": 29.390625, + "grad_norm": 8.702829290996306e-06, + "learning_rate": 1.1644944887324339e-05, + "loss": 1.4537464210206963e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1881, + "tokens/total": 28400768, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 608099 + }, + { + "epoch": 29.40625, + "grad_norm": 1.937136403284967e-05, + "learning_rate": 1.1625538902056175e-05, + "loss": 4.3750281975007965e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1882, + "tokens/total": 28415948, + "tokens/train_per_sec_per_gpu": 9.0, + "tokens/trainable": 608423 + }, + { + "epoch": 29.421875, + "grad_norm": 1.0579096851870418e-05, + "learning_rate": 1.1606245960932421e-05, + "loss": 2.0813200762859196e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1883, + "tokens/total": 28431112, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 608745 + }, + { + "epoch": 29.4375, + "grad_norm": 1.0840181857929565e-05, + "learning_rate": 1.1587066114235063e-05, + "loss": 2.3438825280663877e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1884, + "tokens/total": 28446276, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 609068 + }, + { + "epoch": 29.453125, + "grad_norm": 7.742099114693701e-06, + "learning_rate": 1.1567999411951337e-05, + "loss": 1.5444898338046187e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1885, + "tokens/total": 28461448, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 609397 + }, + { + "epoch": 29.46875, + "grad_norm": 4.0944956708699465e-05, + "learning_rate": 1.154904590377359e-05, + "loss": 4.129812509745534e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1886, + "tokens/total": 28476492, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 609723 + }, + { + "epoch": 29.484375, + "grad_norm": 1.2082108696631622e-05, + "learning_rate": 1.1530205639099166e-05, + "loss": 1.1788269915768979e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1887, + "tokens/total": 28491620, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 610046 + }, + { + "epoch": 29.5, + "grad_norm": 1.2721752682409715e-05, + "learning_rate": 1.151147866703028e-05, + "loss": 2.7839803351525916e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1888, + "tokens/total": 28506804, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 610368 + }, + { + "epoch": 29.515625, + "grad_norm": 4.812082352145808e-06, + "learning_rate": 1.1492865036373855e-05, + "loss": 8.53410639933827e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1889, + "tokens/total": 28521888, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 610696 + }, + { + "epoch": 29.53125, + "grad_norm": 7.576979896839475e-06, + "learning_rate": 1.1474364795641428e-05, + "loss": 1.642981715122005e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1890, + "tokens/total": 28537140, + "tokens/train_per_sec_per_gpu": 9.45, + "tokens/trainable": 611017 + }, + { + "epoch": 29.546875, + "grad_norm": 1.49277566379169e-05, + "learning_rate": 1.1455977993049036e-05, + "loss": 3.2064505717244174e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1891, + "tokens/total": 28552232, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 611339 + }, + { + "epoch": 29.5625, + "grad_norm": 2.1991065295878798e-05, + "learning_rate": 1.1437704676517041e-05, + "loss": 4.202682930554147e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1892, + "tokens/total": 28567348, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 611662 + }, + { + "epoch": 29.578125, + "grad_norm": 2.0108893295400776e-05, + "learning_rate": 1.141954489367005e-05, + "loss": 3.0747099799555144e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1893, + "tokens/total": 28582492, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 611986 + }, + { + "epoch": 29.59375, + "grad_norm": 6.332628800009843e-06, + "learning_rate": 1.1401498691836757e-05, + "loss": 1.234963349361351e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1894, + "tokens/total": 28597584, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 612315 + }, + { + "epoch": 29.609375, + "grad_norm": 1.3458281500788871e-05, + "learning_rate": 1.1383566118049863e-05, + "loss": 2.3035896390410926e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1895, + "tokens/total": 28612636, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 612639 + }, + { + "epoch": 29.625, + "grad_norm": 3.5845107049681246e-05, + "learning_rate": 1.1365747219045907e-05, + "loss": 5.392915340962645e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1896, + "tokens/total": 28627708, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 612965 + }, + { + "epoch": 29.640625, + "grad_norm": 2.8963604563614354e-05, + "learning_rate": 1.1348042041265174e-05, + "loss": 5.824247750751965e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1897, + "tokens/total": 28642880, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 613291 + }, + { + "epoch": 29.65625, + "grad_norm": 9.007120752357878e-06, + "learning_rate": 1.1330450630851554e-05, + "loss": 1.8031677484486863e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1898, + "tokens/total": 28658004, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 613613 + }, + { + "epoch": 29.671875, + "grad_norm": 2.1353138436097652e-05, + "learning_rate": 1.1312973033652449e-05, + "loss": 2.546228756727942e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1899, + "tokens/total": 28673044, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 613936 + }, + { + "epoch": 29.6875, + "grad_norm": 1.516917473054491e-05, + "learning_rate": 1.1295609295218621e-05, + "loss": 2.6923538598566665e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1900, + "tokens/total": 28688188, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 614257 + }, + { + "epoch": 29.703125, + "grad_norm": 9.172044519800693e-05, + "learning_rate": 1.1278359460804097e-05, + "loss": 8.048674544625101e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1901, + "tokens/total": 28703332, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 614578 + }, + { + "epoch": 29.71875, + "grad_norm": 7.274218205566285e-06, + "learning_rate": 1.126122357536604e-05, + "loss": 1.2520632708401536e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1902, + "tokens/total": 28718344, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 614900 + }, + { + "epoch": 29.734375, + "grad_norm": 1.1625887964328285e-05, + "learning_rate": 1.124420168356463e-05, + "loss": 2.1893134771744371e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1903, + "tokens/total": 28733420, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 615224 + }, + { + "epoch": 29.75, + "grad_norm": 1.6474876247229986e-05, + "learning_rate": 1.1227293829762976e-05, + "loss": 2.488126256139367e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1904, + "tokens/total": 28748388, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 615549 + }, + { + "epoch": 29.765625, + "grad_norm": 2.2937243556953035e-05, + "learning_rate": 1.1210500058026946e-05, + "loss": 3.7809635955454723e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1905, + "tokens/total": 28763480, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 615868 + }, + { + "epoch": 29.78125, + "grad_norm": 1.217008048115531e-05, + "learning_rate": 1.11938204121251e-05, + "loss": 2.5232222355953127e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1906, + "tokens/total": 28778696, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 616188 + }, + { + "epoch": 29.796875, + "grad_norm": 9.25617860048078e-06, + "learning_rate": 1.1177254935528556e-05, + "loss": 1.9791629313203885e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1907, + "tokens/total": 28793744, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 616507 + }, + { + "epoch": 29.8125, + "grad_norm": 9.23365314520197e-06, + "learning_rate": 1.1160803671410877e-05, + "loss": 1.4962900252157851e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1908, + "tokens/total": 28808884, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 616828 + }, + { + "epoch": 29.828125, + "grad_norm": 1.0611658581183292e-05, + "learning_rate": 1.1144466662647961e-05, + "loss": 2.1851536757822032e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1909, + "tokens/total": 28823968, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 617148 + }, + { + "epoch": 29.84375, + "grad_norm": 1.5041220649436582e-05, + "learning_rate": 1.1128243951817937e-05, + "loss": 3.348387451751478e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1910, + "tokens/total": 28838976, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 617471 + }, + { + "epoch": 29.859375, + "grad_norm": 1.1446321877883747e-05, + "learning_rate": 1.1112135581201048e-05, + "loss": 1.9469906931135483e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1911, + "tokens/total": 28853956, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 617793 + }, + { + "epoch": 29.875, + "grad_norm": 1.077484012057539e-05, + "learning_rate": 1.1096141592779522e-05, + "loss": 2.183214604656314e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1912, + "tokens/total": 28869048, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 618120 + }, + { + "epoch": 29.890625, + "grad_norm": 2.068456342385616e-05, + "learning_rate": 1.1080262028237489e-05, + "loss": 2.9887812047491025e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1913, + "tokens/total": 28884128, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 618440 + }, + { + "epoch": 29.90625, + "grad_norm": 9.690223123470787e-06, + "learning_rate": 1.1064496928960872e-05, + "loss": 1.6506479028066678e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1914, + "tokens/total": 28899212, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 618766 + }, + { + "epoch": 29.921875, + "grad_norm": 3.450577423791401e-05, + "learning_rate": 1.104884633603726e-05, + "loss": 4.7052580498530006e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1915, + "tokens/total": 28914220, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 619085 + }, + { + "epoch": 29.9375, + "grad_norm": 5.1639010052895173e-05, + "learning_rate": 1.1033310290255813e-05, + "loss": 8.422534847341012e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1916, + "tokens/total": 28929316, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 619408 + }, + { + "epoch": 29.953125, + "grad_norm": 9.160447916656267e-06, + "learning_rate": 1.1017888832107153e-05, + "loss": 1.888603691213575e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1917, + "tokens/total": 28944400, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 619731 + }, + { + "epoch": 29.96875, + "grad_norm": 3.5912558814743534e-05, + "learning_rate": 1.1002582001783273e-05, + "loss": 2.2969335589095863e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1918, + "tokens/total": 28959544, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 620055 + }, + { + "epoch": 29.984375, + "grad_norm": 8.250247446994763e-06, + "learning_rate": 1.0987389839177397e-05, + "loss": 1.721806910381929e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 1919, + "tokens/total": 28974588, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 620377 + }, + { + "epoch": 30.0, + "grad_norm": 1.8300093870493583e-05, + "learning_rate": 1.097231238388391e-05, + "loss": 4.083811973032425e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1920, + "tokens/total": 28989656, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 620700 + }, + { + "epoch": 30.015625, + "grad_norm": 1.0169062989007216e-05, + "learning_rate": 1.0957349675198241e-05, + "loss": 2.0245731491286278e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1921, + "tokens/total": 29004788, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 621021 + }, + { + "epoch": 30.03125, + "grad_norm": 1.3734853382629808e-05, + "learning_rate": 1.0942501752116758e-05, + "loss": 2.803357403990958e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1922, + "tokens/total": 29019884, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 621338 + }, + { + "epoch": 30.046875, + "grad_norm": 1.5605177395627834e-05, + "learning_rate": 1.0927768653336678e-05, + "loss": 3.5272753962090064e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1923, + "tokens/total": 29035048, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 621665 + }, + { + "epoch": 30.0625, + "grad_norm": 9.824600056163035e-06, + "learning_rate": 1.0913150417255957e-05, + "loss": 1.7884028125081386e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0, + "step": 1924, + "tokens/total": 29050072, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 621986 + }, + { + "epoch": 30.078125, + "grad_norm": 7.567050488432869e-06, + "learning_rate": 1.0898647081973187e-05, + "loss": 1.237241349372198e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1925, + "tokens/total": 29065208, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 622308 + }, + { + "epoch": 30.09375, + "grad_norm": 3.0804865673417225e-05, + "learning_rate": 1.0884258685287498e-05, + "loss": 5.978179160592845e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1926, + "tokens/total": 29080328, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 622630 + }, + { + "epoch": 30.109375, + "grad_norm": 1.7351077985949814e-05, + "learning_rate": 1.0869985264698477e-05, + "loss": 2.527904996441066e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1927, + "tokens/total": 29095452, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 622957 + }, + { + "epoch": 30.125, + "grad_norm": 1.0766555533336941e-05, + "learning_rate": 1.0855826857406036e-05, + "loss": 2.2809507527199457e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1928, + "tokens/total": 29110532, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 623281 + }, + { + "epoch": 30.140625, + "grad_norm": 1.0759512406366412e-05, + "learning_rate": 1.0841783500310361e-05, + "loss": 2.2200043758857646e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1929, + "tokens/total": 29125644, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 623601 + }, + { + "epoch": 30.15625, + "grad_norm": 1.215000702359248e-05, + "learning_rate": 1.0827855230011758e-05, + "loss": 2.231722362466826e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1930, + "tokens/total": 29140640, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 623924 + }, + { + "epoch": 30.171875, + "grad_norm": 9.377637979923747e-06, + "learning_rate": 1.0814042082810625e-05, + "loss": 1.797453847984798e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1931, + "tokens/total": 29155720, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 624247 + }, + { + "epoch": 30.1875, + "grad_norm": 8.82369749888312e-06, + "learning_rate": 1.0800344094707283e-05, + "loss": 1.1490568141425683e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1932, + "tokens/total": 29170876, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 624571 + }, + { + "epoch": 30.203125, + "grad_norm": 1.8109982192981988e-05, + "learning_rate": 1.0786761301401964e-05, + "loss": 3.1755376994624385e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1933, + "tokens/total": 29186040, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 624891 + }, + { + "epoch": 30.21875, + "grad_norm": 1.2180717021692544e-05, + "learning_rate": 1.0773293738294627e-05, + "loss": 2.4256672759292996e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1934, + "tokens/total": 29201004, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 625212 + }, + { + "epoch": 30.234375, + "grad_norm": 2.831067831721157e-05, + "learning_rate": 1.0759941440484967e-05, + "loss": 5.704957857233239e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1935, + "tokens/total": 29216160, + "tokens/train_per_sec_per_gpu": 8.84, + "tokens/trainable": 625535 + }, + { + "epoch": 30.25, + "grad_norm": 2.4006283638300374e-05, + "learning_rate": 1.0746704442772222e-05, + "loss": 3.52918931412205e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1936, + "tokens/total": 29231316, + "tokens/train_per_sec_per_gpu": 9.01, + "tokens/trainable": 625855 + }, + { + "epoch": 30.265625, + "grad_norm": 3.5973553167423233e-05, + "learning_rate": 1.0733582779655157e-05, + "loss": 5.346579996512446e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.0, + "step": 1937, + "tokens/total": 29246604, + "tokens/train_per_sec_per_gpu": 9.21, + "tokens/trainable": 626176 + }, + { + "epoch": 30.28125, + "grad_norm": 5.678681918652728e-05, + "learning_rate": 1.0720576485331943e-05, + "loss": 7.030010920061613e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1938, + "tokens/total": 29261712, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 626501 + }, + { + "epoch": 30.296875, + "grad_norm": 1.430520751455333e-05, + "learning_rate": 1.0707685593700068e-05, + "loss": 3.080564283663989e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 1939, + "tokens/total": 29276760, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 626826 + }, + { + "epoch": 30.3125, + "grad_norm": 1.9665152649395168e-05, + "learning_rate": 1.0694910138356263e-05, + "loss": 4.068976409143943e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1940, + "tokens/total": 29291932, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 627149 + }, + { + "epoch": 30.328125, + "grad_norm": 1.205372427648399e-05, + "learning_rate": 1.068225015259639e-05, + "loss": 1.957125448370789e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1941, + "tokens/total": 29307036, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 627476 + }, + { + "epoch": 30.34375, + "grad_norm": 1.885861092887353e-05, + "learning_rate": 1.0669705669415384e-05, + "loss": 3.5344734783393505e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1942, + "tokens/total": 29322052, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 627796 + }, + { + "epoch": 30.359375, + "grad_norm": 2.217328619735781e-05, + "learning_rate": 1.0657276721507142e-05, + "loss": 2.8971118126719375e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.0, + "step": 1943, + "tokens/total": 29337140, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 628117 + }, + { + "epoch": 30.375, + "grad_norm": 1.1275132237642538e-05, + "learning_rate": 1.0644963341264458e-05, + "loss": 2.0679371459664253e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1944, + "tokens/total": 29352196, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 628443 + }, + { + "epoch": 30.390625, + "grad_norm": 2.3426955522154458e-05, + "learning_rate": 1.0632765560778915e-05, + "loss": 3.822025576027954e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1945, + "tokens/total": 29367204, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 628773 + }, + { + "epoch": 30.40625, + "grad_norm": 1.6201236576307565e-05, + "learning_rate": 1.0620683411840844e-05, + "loss": 3.170169406985224e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1946, + "tokens/total": 29382268, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 629101 + }, + { + "epoch": 30.421875, + "grad_norm": 1.1459446795925032e-05, + "learning_rate": 1.0608716925939172e-05, + "loss": 2.1178207987304631e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1947, + "tokens/total": 29397348, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 629424 + }, + { + "epoch": 30.4375, + "grad_norm": 1.6873536878847517e-05, + "learning_rate": 1.0596866134261421e-05, + "loss": 1.7832043397447706e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1948, + "tokens/total": 29412404, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 629749 + }, + { + "epoch": 30.453125, + "grad_norm": 2.0344697986729443e-05, + "learning_rate": 1.058513106769355e-05, + "loss": 3.65539534641357e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1949, + "tokens/total": 29427336, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 630071 + }, + { + "epoch": 30.46875, + "grad_norm": 1.4918067790858913e-05, + "learning_rate": 1.0573511756819949e-05, + "loss": 2.944382231362397e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1950, + "tokens/total": 29442544, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 630395 + }, + { + "epoch": 30.484375, + "grad_norm": 0.00014585682947654277, + "learning_rate": 1.0562008231923284e-05, + "loss": 1.3316008562469506e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1951, + "tokens/total": 29457600, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 630719 + }, + { + "epoch": 30.5, + "grad_norm": 1.6891237464733422e-05, + "learning_rate": 1.0550620522984483e-05, + "loss": 2.874154461096623e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1952, + "tokens/total": 29472684, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 631041 + }, + { + "epoch": 30.515625, + "grad_norm": 6.828726782259764e-06, + "learning_rate": 1.053934865968261e-05, + "loss": 1.349200857703181e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 1953, + "tokens/total": 29487708, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 631365 + }, + { + "epoch": 30.53125, + "grad_norm": 9.460346336709335e-05, + "learning_rate": 1.0528192671394826e-05, + "loss": 8.776854087955144e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1954, + "tokens/total": 29502820, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 631688 + }, + { + "epoch": 30.546875, + "grad_norm": 7.713179911661427e-06, + "learning_rate": 1.0517152587196279e-05, + "loss": 1.5734350711227307e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1955, + "tokens/total": 29518012, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 632009 + }, + { + "epoch": 30.5625, + "grad_norm": 1.480219179939013e-05, + "learning_rate": 1.0506228435860055e-05, + "loss": 2.4792416297714226e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1956, + "tokens/total": 29533048, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 632328 + }, + { + "epoch": 30.578125, + "grad_norm": 6.468358151323628e-06, + "learning_rate": 1.0495420245857086e-05, + "loss": 1.1547638933961935e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1957, + "tokens/total": 29548156, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 632658 + }, + { + "epoch": 30.59375, + "grad_norm": 1.3387553735810798e-05, + "learning_rate": 1.0484728045356076e-05, + "loss": 2.5908511247507704e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1958, + "tokens/total": 29563248, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 632982 + }, + { + "epoch": 30.609375, + "grad_norm": 2.5314027880085632e-05, + "learning_rate": 1.0474151862223446e-05, + "loss": 4.5952720029163174e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1959, + "tokens/total": 29578392, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 633305 + }, + { + "epoch": 30.625, + "grad_norm": 1.3550347830459941e-05, + "learning_rate": 1.0463691724023228e-05, + "loss": 1.9667382389343402e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1960, + "tokens/total": 29593432, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 633630 + }, + { + "epoch": 30.640625, + "grad_norm": 3.8178714021341875e-05, + "learning_rate": 1.045334765801704e-05, + "loss": 2.602246809146891e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1961, + "tokens/total": 29608492, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 633953 + }, + { + "epoch": 30.65625, + "grad_norm": 7.339458534261212e-05, + "learning_rate": 1.0443119691163966e-05, + "loss": 9.592055221219198e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1962, + "tokens/total": 29623732, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 634277 + }, + { + "epoch": 30.671875, + "grad_norm": 1.0366945389250759e-05, + "learning_rate": 1.0433007850120519e-05, + "loss": 1.9831104225431773e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1963, + "tokens/total": 29638924, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 634597 + }, + { + "epoch": 30.6875, + "grad_norm": 6.617826784349745e-06, + "learning_rate": 1.042301216124055e-05, + "loss": 1.1860656456974539e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1964, + "tokens/total": 29654004, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 634919 + }, + { + "epoch": 30.703125, + "grad_norm": 4.1820067053777166e-06, + "learning_rate": 1.0413132650575213e-05, + "loss": 7.786368882989336e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 1965, + "tokens/total": 29669244, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 635245 + }, + { + "epoch": 30.71875, + "grad_norm": 1.1503453606565017e-05, + "learning_rate": 1.0403369343872845e-05, + "loss": 2.3612352606505738e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1966, + "tokens/total": 29684384, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 635568 + }, + { + "epoch": 30.734375, + "grad_norm": 6.660868621111149e-06, + "learning_rate": 1.0393722266578948e-05, + "loss": 1.2692035511463473e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1967, + "tokens/total": 29699476, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 635893 + }, + { + "epoch": 30.75, + "grad_norm": 9.04143598745577e-06, + "learning_rate": 1.0384191443836093e-05, + "loss": 1.3700594081456074e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 1968, + "tokens/total": 29714576, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 636215 + }, + { + "epoch": 30.765625, + "grad_norm": 1.648299439693801e-05, + "learning_rate": 1.0374776900483867e-05, + "loss": 2.812823254316754e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1969, + "tokens/total": 29729600, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 636535 + }, + { + "epoch": 30.78125, + "grad_norm": 7.042687229841249e-06, + "learning_rate": 1.036547866105881e-05, + "loss": 1.3705098922400794e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1970, + "tokens/total": 29744708, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 636858 + }, + { + "epoch": 30.796875, + "grad_norm": 2.9748962333542295e-05, + "learning_rate": 1.0356296749794338e-05, + "loss": 6.321104706330516e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 1971, + "tokens/total": 29759712, + "tokens/train_per_sec_per_gpu": 9.87, + "tokens/trainable": 637183 + }, + { + "epoch": 30.8125, + "grad_norm": 8.364554560102988e-06, + "learning_rate": 1.0347231190620688e-05, + "loss": 1.6406795566581422e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 33.0, + "memory/max_allocated (GiB)": 33.0, + "ppl": 1.0, + "step": 1972, + "tokens/total": 29774784, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 637504 + }, + { + "epoch": 30.828125, + "grad_norm": 3.532248592819087e-05, + "learning_rate": 1.0338282007164868e-05, + "loss": 4.589630862028571e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1973, + "tokens/total": 29789852, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 637826 + }, + { + "epoch": 30.84375, + "grad_norm": 1.0972508789564017e-05, + "learning_rate": 1.0329449222750567e-05, + "loss": 2.662062286162836e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1974, + "tokens/total": 29804892, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 638150 + }, + { + "epoch": 30.859375, + "grad_norm": 6.266405762289651e-06, + "learning_rate": 1.0320732860398126e-05, + "loss": 1.2258912818197132e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1975, + "tokens/total": 29820144, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 638476 + }, + { + "epoch": 30.875, + "grad_norm": 1.1950245607295074e-05, + "learning_rate": 1.031213294282445e-05, + "loss": 2.241455661078362e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1976, + "tokens/total": 29835260, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 638799 + }, + { + "epoch": 30.890625, + "grad_norm": 1.1543971595529001e-05, + "learning_rate": 1.0303649492442971e-05, + "loss": 2.4778694296401227e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1977, + "tokens/total": 29850364, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 639121 + }, + { + "epoch": 30.90625, + "grad_norm": 2.4364357159356587e-05, + "learning_rate": 1.029528253136357e-05, + "loss": 3.05993239635427e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1978, + "tokens/total": 29865496, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 639446 + }, + { + "epoch": 30.921875, + "grad_norm": 2.2705880837747827e-05, + "learning_rate": 1.028703208139253e-05, + "loss": 3.4097857337656023e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1979, + "tokens/total": 29880492, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 639772 + }, + { + "epoch": 30.9375, + "grad_norm": 1.212633833347354e-05, + "learning_rate": 1.0278898164032491e-05, + "loss": 2.1989676213252096e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1980, + "tokens/total": 29895648, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 640098 + }, + { + "epoch": 30.953125, + "grad_norm": 1.1980173439951614e-05, + "learning_rate": 1.0270880800482372e-05, + "loss": 2.5809973180912493e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 1981, + "tokens/total": 29910740, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 640424 + }, + { + "epoch": 30.96875, + "grad_norm": 6.24211543254205e-06, + "learning_rate": 1.0262980011637325e-05, + "loss": 1.1765812502062545e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 1982, + "tokens/total": 29925904, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 640745 + }, + { + "epoch": 30.984375, + "grad_norm": 1.5217317923088558e-05, + "learning_rate": 1.0255195818088677e-05, + "loss": 2.9517323696381936e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 1983, + "tokens/total": 29941016, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 641070 + }, + { + "epoch": 31.0, + "grad_norm": 2.0580229829647578e-05, + "learning_rate": 1.0247528240123894e-05, + "loss": 4.4977525703870924e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.0, + "step": 1984, + "tokens/total": 29955968, + "tokens/train_per_sec_per_gpu": 9.65, + "tokens/trainable": 641390 + }, + { + "epoch": 31.015625, + "grad_norm": 2.3435582988895476e-05, + "learning_rate": 1.0239977297726498e-05, + "loss": 3.9014616959320847e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1985, + "tokens/total": 29971064, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 641717 + }, + { + "epoch": 31.03125, + "grad_norm": 7.909923624538351e-06, + "learning_rate": 1.0232543010576036e-05, + "loss": 1.5270512676579528e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1986, + "tokens/total": 29986064, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 642035 + }, + { + "epoch": 31.046875, + "grad_norm": 1.2623117072507739e-05, + "learning_rate": 1.0225225398048028e-05, + "loss": 2.179548488356886e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1987, + "tokens/total": 30001116, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 642358 + }, + { + "epoch": 31.0625, + "grad_norm": 1.2466032785596326e-05, + "learning_rate": 1.0218024479213911e-05, + "loss": 2.1969893282403063e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1988, + "tokens/total": 30016220, + "tokens/train_per_sec_per_gpu": 8.82, + "tokens/trainable": 642677 + }, + { + "epoch": 31.078125, + "grad_norm": 1.4325033589557279e-05, + "learning_rate": 1.0210940272840991e-05, + "loss": 2.6749245307655656e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1989, + "tokens/total": 30031380, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 643003 + }, + { + "epoch": 31.09375, + "grad_norm": 1.3875001059204806e-05, + "learning_rate": 1.0203972797392391e-05, + "loss": 1.873444830380322e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1990, + "tokens/total": 30046396, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 643327 + }, + { + "epoch": 31.109375, + "grad_norm": 1.0614858183544129e-05, + "learning_rate": 1.0197122071026999e-05, + "loss": 2.234033331660612e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 1991, + "tokens/total": 30061396, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 643648 + }, + { + "epoch": 31.125, + "grad_norm": 1.0526632649998646e-05, + "learning_rate": 1.0190388111599446e-05, + "loss": 1.8647936883553484e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 1992, + "tokens/total": 30076380, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 643967 + }, + { + "epoch": 31.140625, + "grad_norm": 1.0613497579470277e-05, + "learning_rate": 1.0183770936660021e-05, + "loss": 1.9999818334781594e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1993, + "tokens/total": 30091484, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 644290 + }, + { + "epoch": 31.15625, + "grad_norm": 1.0127177120011766e-05, + "learning_rate": 1.0177270563454658e-05, + "loss": 2.1199551270001393e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1994, + "tokens/total": 30106668, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 644614 + }, + { + "epoch": 31.171875, + "grad_norm": 6.413915343728149e-06, + "learning_rate": 1.017088700892487e-05, + "loss": 1.3457959369134187e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1995, + "tokens/total": 30121836, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 644934 + }, + { + "epoch": 31.1875, + "grad_norm": 3.1155432225205004e-05, + "learning_rate": 1.0164620289707715e-05, + "loss": 5.196276333663263e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 1996, + "tokens/total": 30136764, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 645256 + }, + { + "epoch": 31.203125, + "grad_norm": 1.9350174625287764e-05, + "learning_rate": 1.0158470422135751e-05, + "loss": 3.049296992685413e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 1997, + "tokens/total": 30151912, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 645582 + }, + { + "epoch": 31.21875, + "grad_norm": 8.49901061883429e-06, + "learning_rate": 1.0152437422236999e-05, + "loss": 1.7272982688609773e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 1998, + "tokens/total": 30166936, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 645907 + }, + { + "epoch": 31.234375, + "grad_norm": 9.765965842234436e-06, + "learning_rate": 1.0146521305734883e-05, + "loss": 1.4377866364156944e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 1999, + "tokens/total": 30181996, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 646237 + }, + { + "epoch": 31.25, + "grad_norm": 6.864878741907887e-06, + "learning_rate": 1.0140722088048205e-05, + "loss": 1.272786249728597e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2000, + "tokens/total": 30197008, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 646560 + }, + { + "epoch": 31.265625, + "grad_norm": 2.8129066777182743e-05, + "learning_rate": 1.0135039784291113e-05, + "loss": 3.8548969882867823e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2001, + "tokens/total": 30212100, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 646884 + }, + { + "epoch": 31.28125, + "grad_norm": 2.2618229195359163e-05, + "learning_rate": 1.0129474409273033e-05, + "loss": 4.773054342877003e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2002, + "tokens/total": 30227228, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 647204 + }, + { + "epoch": 31.296875, + "grad_norm": 9.661648618930485e-06, + "learning_rate": 1.0124025977498657e-05, + "loss": 1.515105481075807e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 2003, + "tokens/total": 30242208, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 647527 + }, + { + "epoch": 31.3125, + "grad_norm": 1.5448891645064577e-05, + "learning_rate": 1.0118694503167885e-05, + "loss": 3.0167876730047283e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 2004, + "tokens/total": 30257356, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 647852 + }, + { + "epoch": 31.328125, + "grad_norm": 0.00016340894217137247, + "learning_rate": 1.0113480000175818e-05, + "loss": 1.3371639795423107e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2005, + "tokens/total": 30272452, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 648176 + }, + { + "epoch": 31.34375, + "grad_norm": 2.5538760382914916e-05, + "learning_rate": 1.0108382482112672e-05, + "loss": 3.8856595097058744e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2006, + "tokens/total": 30287584, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 648497 + }, + { + "epoch": 31.359375, + "grad_norm": 3.380901398486458e-05, + "learning_rate": 1.0103401962263806e-05, + "loss": 4.870926773037354e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2007, + "tokens/total": 30302644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 648816 + }, + { + "epoch": 31.375, + "grad_norm": 1.8713415556703694e-05, + "learning_rate": 1.0098538453609624e-05, + "loss": 2.2138090116641251e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2008, + "tokens/total": 30317716, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 649140 + }, + { + "epoch": 31.390625, + "grad_norm": 8.986103239294607e-06, + "learning_rate": 1.009379196882559e-05, + "loss": 1.6187237861231552e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 2009, + "tokens/total": 30332832, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 649464 + }, + { + "epoch": 31.40625, + "grad_norm": 8.772605360718444e-05, + "learning_rate": 1.0089162520282172e-05, + "loss": 8.986606871985714e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2010, + "tokens/total": 30347944, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 649784 + }, + { + "epoch": 31.421875, + "grad_norm": 2.0709696400444955e-05, + "learning_rate": 1.0084650120044816e-05, + "loss": 3.4837080420402344e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2011, + "tokens/total": 30363124, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 650108 + }, + { + "epoch": 31.4375, + "grad_norm": 9.576235243002884e-06, + "learning_rate": 1.0080254779873893e-05, + "loss": 2.0411842172052275e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2012, + "tokens/total": 30378296, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 650433 + }, + { + "epoch": 31.453125, + "grad_norm": 2.6904082915280014e-05, + "learning_rate": 1.0075976511224718e-05, + "loss": 5.955585038464051e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2013, + "tokens/total": 30393396, + "tokens/train_per_sec_per_gpu": 10.0, + "tokens/trainable": 650759 + }, + { + "epoch": 31.46875, + "grad_norm": 7.172536425059661e-05, + "learning_rate": 1.0071815325247474e-05, + "loss": 9.976895398722263e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 2014, + "tokens/total": 30408500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 651078 + }, + { + "epoch": 31.484375, + "grad_norm": 3.6888573959004134e-05, + "learning_rate": 1.006777123278719e-05, + "loss": 4.0864063066692324e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2015, + "tokens/total": 30423568, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 651399 + }, + { + "epoch": 31.5, + "grad_norm": 1.1561081919353455e-05, + "learning_rate": 1.0063844244383736e-05, + "loss": 2.5746334131326876e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2016, + "tokens/total": 30438648, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 651720 + }, + { + "epoch": 31.515625, + "grad_norm": 8.27141684567323e-06, + "learning_rate": 1.006003437027178e-05, + "loss": 1.7990186051974888e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2017, + "tokens/total": 30453668, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 652046 + }, + { + "epoch": 31.53125, + "grad_norm": 9.241395673598163e-06, + "learning_rate": 1.0056341620380752e-05, + "loss": 1.8860107786622393e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2018, + "tokens/total": 30468808, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 652370 + }, + { + "epoch": 31.546875, + "grad_norm": 1.0602431757433806e-05, + "learning_rate": 1.0052766004334845e-05, + "loss": 1.8454200301221135e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 2019, + "tokens/total": 30483776, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 652694 + }, + { + "epoch": 31.5625, + "grad_norm": 8.238218470069114e-06, + "learning_rate": 1.004930753145295e-05, + "loss": 1.5227141147988732e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2020, + "tokens/total": 30498864, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 653019 + }, + { + "epoch": 31.578125, + "grad_norm": 1.5514528058702126e-05, + "learning_rate": 1.004596621074868e-05, + "loss": 2.6144121534343867e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2021, + "tokens/total": 30514044, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 653341 + }, + { + "epoch": 31.59375, + "grad_norm": 9.879069693852216e-06, + "learning_rate": 1.0042742050930303e-05, + "loss": 1.9487379177007824e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2022, + "tokens/total": 30529208, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 653661 + }, + { + "epoch": 31.609375, + "grad_norm": 7.210634066723287e-06, + "learning_rate": 1.0039635060400747e-05, + "loss": 1.414278756328713e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 2023, + "tokens/total": 30544296, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 653983 + }, + { + "epoch": 31.625, + "grad_norm": 1.4184544852469116e-05, + "learning_rate": 1.0036645247257572e-05, + "loss": 2.0518878329767176e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2024, + "tokens/total": 30559248, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 654306 + }, + { + "epoch": 31.640625, + "grad_norm": 9.930431588145439e-06, + "learning_rate": 1.0033772619292936e-05, + "loss": 2.040439710526698e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2025, + "tokens/total": 30574408, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 654634 + }, + { + "epoch": 31.65625, + "grad_norm": 9.48280194279505e-06, + "learning_rate": 1.0031017183993589e-05, + "loss": 1.7529664830817637e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 2026, + "tokens/total": 30589532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 654963 + }, + { + "epoch": 31.671875, + "grad_norm": 2.440475691400934e-05, + "learning_rate": 1.0028378948540847e-05, + "loss": 3.7373831673903624e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2027, + "tokens/total": 30604664, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 655285 + }, + { + "epoch": 31.6875, + "grad_norm": 0.00014577741967514157, + "learning_rate": 1.0025857919810583e-05, + "loss": 1.4009040114615345e-06, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2028, + "tokens/total": 30619804, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 655610 + }, + { + "epoch": 31.703125, + "grad_norm": 1.0694270713429432e-05, + "learning_rate": 1.0023454104373196e-05, + "loss": 2.16418968079779e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2029, + "tokens/total": 30634832, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 655938 + }, + { + "epoch": 31.71875, + "grad_norm": 1.9057873942074366e-05, + "learning_rate": 1.0021167508493605e-05, + "loss": 4.054859346069861e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2030, + "tokens/total": 30650048, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 656261 + }, + { + "epoch": 31.734375, + "grad_norm": 1.2024561328871641e-05, + "learning_rate": 1.0018998138131207e-05, + "loss": 2.45875412474561e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0, + "step": 2031, + "tokens/total": 30665232, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 656581 + }, + { + "epoch": 31.75, + "grad_norm": 1.3288386981002986e-05, + "learning_rate": 1.0016945998939908e-05, + "loss": 2.619566998873779e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2032, + "tokens/total": 30680472, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 656908 + }, + { + "epoch": 31.765625, + "grad_norm": 3.19693140227173e-06, + "learning_rate": 1.001501109626807e-05, + "loss": 6.125334550688422e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2033, + "tokens/total": 30695508, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 657230 + }, + { + "epoch": 31.78125, + "grad_norm": 4.827685916097835e-05, + "learning_rate": 1.0013193435158503e-05, + "loss": 6.761121085219202e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 2034, + "tokens/total": 30710592, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 657554 + }, + { + "epoch": 31.796875, + "grad_norm": 1.7207978089572862e-05, + "learning_rate": 1.001149302034847e-05, + "loss": 3.5054395652878156e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0, + "step": 2035, + "tokens/total": 30725724, + "tokens/train_per_sec_per_gpu": 10.17, + "tokens/trainable": 657876 + }, + { + "epoch": 31.8125, + "grad_norm": 9.439231689611915e-06, + "learning_rate": 1.0009909856269647e-05, + "loss": 1.9994857325400517e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 2036, + "tokens/total": 30740820, + "tokens/train_per_sec_per_gpu": 8.76, + "tokens/trainable": 658192 + }, + { + "epoch": 31.828125, + "grad_norm": 2.103359111060854e-05, + "learning_rate": 1.0008443947048141e-05, + "loss": 4.648667584206123e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 2037, + "tokens/total": 30755904, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 658516 + }, + { + "epoch": 31.84375, + "grad_norm": 9.646611943026073e-06, + "learning_rate": 1.0007095296504465e-05, + "loss": 1.716232844728438e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2038, + "tokens/total": 30771040, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 658838 + }, + { + "epoch": 31.859375, + "grad_norm": 3.643306263256818e-05, + "learning_rate": 1.0005863908153509e-05, + "loss": 5.348643981051282e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 2039, + "tokens/total": 30786180, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 659156 + }, + { + "epoch": 31.875, + "grad_norm": 6.821813258284237e-06, + "learning_rate": 1.0004749785204575e-05, + "loss": 1.2198688636999577e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2040, + "tokens/total": 30801260, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 659479 + }, + { + "epoch": 31.890625, + "grad_norm": 1.857114148151595e-05, + "learning_rate": 1.0003752930561329e-05, + "loss": 2.1997959720465587e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 2041, + "tokens/total": 30816280, + "tokens/train_per_sec_per_gpu": 8.63, + "tokens/trainable": 659804 + }, + { + "epoch": 31.90625, + "grad_norm": 1.107365096686408e-05, + "learning_rate": 1.0002873346821806e-05, + "loss": 2.184269050076182e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2042, + "tokens/total": 30831464, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 660133 + }, + { + "epoch": 31.921875, + "grad_norm": 1.1825974979728926e-05, + "learning_rate": 1.0002111036278416e-05, + "loss": 1.9484311053474812e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2043, + "tokens/total": 30846600, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 660458 + }, + { + "epoch": 31.9375, + "grad_norm": 1.0748093700385652e-05, + "learning_rate": 1.000146600091792e-05, + "loss": 1.9298055065064545e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 2044, + "tokens/total": 30861668, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 660781 + }, + { + "epoch": 31.953125, + "grad_norm": 5.703510396415368e-05, + "learning_rate": 1.0000938242421431e-05, + "loss": 3.039629064005567e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 2045, + "tokens/total": 30876808, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 661108 + }, + { + "epoch": 31.96875, + "grad_norm": 4.361451374279568e-06, + "learning_rate": 1.0000527762164416e-05, + "loss": 7.622089270853394e-08, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.0, + "step": 2046, + "tokens/total": 30892040, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 661434 + }, + { + "epoch": 31.984375, + "grad_norm": 1.314548990194453e-05, + "learning_rate": 1.0000234561216681e-05, + "loss": 2.5190118435602926e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 2047, + "tokens/total": 30907268, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 661759 + }, + { + "epoch": 32.0, + "grad_norm": 1.2614774277608376e-05, + "learning_rate": 1.0000058640342378e-05, + "loss": 2.6136564201806323e-07, + "memory/device_reserved (GiB)": 34.23, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 2048, + "tokens/total": 30922344, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 662080 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1220661885648701e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-2048/training_args.bin b/aft/coin/checkpoint-2048/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-2048/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-256/README.md b/aft/coin/checkpoint-256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-256/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-256/adapter_config.json b/aft/coin/checkpoint-256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-256/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-256/adapter_model.safetensors b/aft/coin/checkpoint-256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..29ce0e1ff11b0b387cc772b468df1a9c7e9210fe --- /dev/null +++ b/aft/coin/checkpoint-256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6541ea1eb82b1ed7118df23abda67a9d35e8ca3310dd43195831655579e118a +size 1047628488 diff --git a/aft/coin/checkpoint-256/chat_template.jinja b/aft/coin/checkpoint-256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-256/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-256/tokenizer.json b/aft/coin/checkpoint-256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-256/tokenizer_config.json b/aft/coin/checkpoint-256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-256/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-256/tokens_state.json b/aft/coin/checkpoint-256/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8c48b36f7d0167098a51fb85b5a56e751a0fc40b --- /dev/null +++ b/aft/coin/checkpoint-256/tokens_state.json @@ -0,0 +1 @@ +{"total": 3865312, "trainable": 82760} \ No newline at end of file diff --git a/aft/coin/checkpoint-256/trainer_state.json b/aft/coin/checkpoint-256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..46eedc4c46e631ee703edbfd285e776c1bd905f1 --- /dev/null +++ b/aft/coin/checkpoint-256/trainer_state.json @@ -0,0 +1,3618 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0028458833694458, + "learning_rate": 6.274509803921569e-05, + "loss": 0.01862957514822483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0188, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.9001771807670593, + "learning_rate": 6.372549019607843e-05, + "loss": 0.021384725347161293, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02162, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.5960713028907776, + "learning_rate": 6.470588235294118e-05, + "loss": 0.010063373483717442, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01011, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 2.038512706756592, + "learning_rate": 6.568627450980392e-05, + "loss": 0.03831503912806511, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03906, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 2.5726208686828613, + "learning_rate": 6.666666666666667e-05, + "loss": 0.03134162724018097, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03184, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.793451189994812, + "learning_rate": 6.764705882352942e-05, + "loss": 0.028099995106458664, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0285, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 0.7262976169586182, + "learning_rate": 6.862745098039216e-05, + "loss": 0.02038084715604782, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02059, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 0.532096803188324, + "learning_rate": 6.96078431372549e-05, + "loss": 0.013697818852961063, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01379, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 0.7747638821601868, + "learning_rate": 7.058823529411765e-05, + "loss": 0.023866616189479828, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02415, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 0.49953746795654297, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014121961779892445, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01422, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.3340316116809845, + "learning_rate": 7.254901960784314e-05, + "loss": 0.012052969075739384, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01213, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.735864520072937, + "learning_rate": 7.352941176470589e-05, + "loss": 0.04014549404382706, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04096, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.4149185419082642, + "learning_rate": 7.450980392156864e-05, + "loss": 0.045495182275772095, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04655, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 0.31831473112106323, + "learning_rate": 7.549019607843137e-05, + "loss": 0.011568848043680191, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01164, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 0.5228955745697021, + "learning_rate": 7.647058823529411e-05, + "loss": 0.02045898698270321, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02067, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.5053377151489258, + "learning_rate": 7.745098039215687e-05, + "loss": 0.020432129502296448, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02064, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.35163524746894836, + "learning_rate": 7.843137254901961e-05, + "loss": 0.01403958909213543, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01414, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.6905986070632935, + "learning_rate": 7.941176470588235e-05, + "loss": 0.03329383581876755, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.03385, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 0.4459448456764221, + "learning_rate": 8.039215686274511e-05, + "loss": 0.006232030689716339, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00625, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.2638562321662903, + "learning_rate": 8.137254901960785e-05, + "loss": 0.006893642246723175, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00692, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 1.170509696006775, + "learning_rate": 8.23529411764706e-05, + "loss": 0.0398201048374176, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.04062, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.8102952837944031, + "learning_rate": 8.333333333333334e-05, + "loss": 0.018316198140382767, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01848, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.6812735795974731, + "learning_rate": 8.431372549019608e-05, + "loss": 0.02392837405204773, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02422, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.251705288887024, + "learning_rate": 8.529411764705883e-05, + "loss": 0.024196332320570946, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02449, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.5772960782051086, + "learning_rate": 8.627450980392158e-05, + "loss": 0.021560901775956154, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0218, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.6470917463302612, + "learning_rate": 8.725490196078432e-05, + "loss": 0.04491877555847168, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.04594, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.41440942883491516, + "learning_rate": 8.823529411764706e-05, + "loss": 0.0230612400919199, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02333, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 0.39720970392227173, + "learning_rate": 8.921568627450981e-05, + "loss": 0.013313441537320614, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0134, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.8898259401321411, + "learning_rate": 9.019607843137255e-05, + "loss": 0.015421045944094658, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01554, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 8.74992847442627, + "learning_rate": 9.11764705882353e-05, + "loss": 0.03271842002868652, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03326, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.18793171644210815, + "learning_rate": 9.215686274509804e-05, + "loss": 0.0015747153665870428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00158, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.9997888207435608, + "learning_rate": 9.313725490196079e-05, + "loss": 0.03334064409136772, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0339, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.6498851776123047, + "learning_rate": 9.411764705882353e-05, + "loss": 0.023218588903546333, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02349, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.5593264102935791, + "learning_rate": 9.509803921568627e-05, + "loss": 0.005530651658773422, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00555, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.8789476752281189, + "learning_rate": 9.607843137254903e-05, + "loss": 0.04554657265543938, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0466, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.5140370726585388, + "learning_rate": 9.705882352941177e-05, + "loss": 0.014101475477218628, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0142, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 0.5285096764564514, + "learning_rate": 9.80392156862745e-05, + "loss": 0.030611136928200722, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03108, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 0.6092981696128845, + "learning_rate": 9.901960784313727e-05, + "loss": 0.028998689725995064, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02942, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0235849618911743, + "learning_rate": 0.0001, + "loss": 0.025761116296052933, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0261, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.9383920431137085, + "learning_rate": 9.999994135965763e-05, + "loss": 0.02419213019311428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02449, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.8995917439460754, + "learning_rate": 9.999976543878332e-05, + "loss": 0.011181461624801159, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01124, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 3.9120519161224365, + "learning_rate": 9.999947223783559e-05, + "loss": 0.10826049745082855, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.11434, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 1.801079511642456, + "learning_rate": 9.999906175757857e-05, + "loss": 0.029263045638799667, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0297, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 3.040914535522461, + "learning_rate": 9.99985339990821e-05, + "loss": 0.04611361399292946, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04719, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.6034630537033081, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0116285290569067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0117, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.238577961921692, + "learning_rate": 9.999712665317819e-05, + "loss": 0.037302885204553604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03801, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.404407262802124, + "learning_rate": 9.999624706943868e-05, + "loss": 0.010177684016525745, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01023, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 1.1107022762298584, + "learning_rate": 9.999525021479544e-05, + "loss": 0.016156412661075592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01629, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 2.0084729194641113, + "learning_rate": 9.99941360918465e-05, + "loss": 0.022498758509755135, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02275, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.9237216711044312, + "learning_rate": 9.999290470349554e-05, + "loss": 0.009256366640329361, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0093, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 1.0012309551239014, + "learning_rate": 9.999155605295188e-05, + "loss": 0.01096038706600666, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01102, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 0.36062437295913696, + "learning_rate": 9.999009014373037e-05, + "loss": 0.0026956750079989433, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0027, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.09146630764007568, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0006923460750840604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00069, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 0.35573962330818176, + "learning_rate": 9.99868065648415e-05, + "loss": 0.0032197514083236456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00322, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.18824201822280884, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0015725638950243592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00157, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 2.2947869300842285, + "learning_rate": 9.998305400106009e-05, + "loss": 0.09606514871120453, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.10083, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.6851844787597656, + "learning_rate": 9.998100186186879e-05, + "loss": 0.006874916143715382, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0069, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.7458359003067017, + "learning_rate": 9.997883249150642e-05, + "loss": 0.020516997203230858, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02073, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 0.7431558966636658, + "learning_rate": 9.99765458956268e-05, + "loss": 0.014651929959654808, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01476, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.16357871890068054, + "learning_rate": 9.997414208018943e-05, + "loss": 0.003089388832449913, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00309, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.36700522899627686, + "learning_rate": 9.997162105145916e-05, + "loss": 0.01339892391115427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01349, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.22755062580108643, + "learning_rate": 9.996898281600642e-05, + "loss": 0.005444459617137909, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00546, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.339722216129303, + "learning_rate": 9.996622738070706e-05, + "loss": 0.004514522384852171, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00452, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 1.7278125286102295, + "learning_rate": 9.996335475274243e-05, + "loss": 0.024016786366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02431, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.47769469022750854, + "learning_rate": 9.996036493959926e-05, + "loss": 0.010395423509180546, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01045, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.4170791804790497, + "learning_rate": 9.99572579490697e-05, + "loss": 0.004119648598134518, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00413, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.37145885825157166, + "learning_rate": 9.995403378925132e-05, + "loss": 0.00920130219310522, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00924, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 0.06787586212158203, + "learning_rate": 9.995069246854705e-05, + "loss": 0.0012981402687728405, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.7351065874099731, + "learning_rate": 9.994723399566515e-05, + "loss": 0.021887682378292084, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02213, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.590771496295929, + "learning_rate": 9.994365837961926e-05, + "loss": 0.009705535136163235, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00975, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.3608993589878082, + "learning_rate": 9.993996562972823e-05, + "loss": 0.008060690015554428, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00809, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.48873215913772583, + "learning_rate": 9.993615575561626e-05, + "loss": 0.008283237926661968, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00832, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 0.7132001519203186, + "learning_rate": 9.99322287672128e-05, + "loss": 0.024729935452342033, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02504, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.09246033430099487, + "learning_rate": 9.992818467475254e-05, + "loss": 0.001992235891520977, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00199, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.7287611365318298, + "learning_rate": 9.992402348877528e-05, + "loss": 0.021320564672350883, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02155, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 0.2133086621761322, + "learning_rate": 9.991974522012612e-05, + "loss": 0.00337275885976851, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00338, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.37355223298072815, + "learning_rate": 9.99153498799552e-05, + "loss": 0.007823438383638859, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00785, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 0.06703885644674301, + "learning_rate": 9.991083747971783e-05, + "loss": 0.0005649570957757533, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00057, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.06440247595310211, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0006133198039606214, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00061, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 0.34707385301589966, + "learning_rate": 9.990146154639039e-05, + "loss": 0.0035533613990992308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00356, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 2.017024517059326, + "learning_rate": 9.98965980377362e-05, + "loss": 0.042595766484737396, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04352, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.785468339920044, + "learning_rate": 9.989161751788734e-05, + "loss": 0.002102769911289215, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0021, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 2.8202157020568848, + "learning_rate": 9.98865199998242e-05, + "loss": 0.01083361729979515, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01089, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 1.0493052005767822, + "learning_rate": 9.988130549683211e-05, + "loss": 0.014770902693271637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01488, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 2.157977819442749, + "learning_rate": 9.987597402250136e-05, + "loss": 0.009032780304551125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00907, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.029030824080109596, + "learning_rate": 9.987052559072696e-05, + "loss": 0.00013317995762918144, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 2.6754238605499268, + "learning_rate": 9.98649602157089e-05, + "loss": 0.033499784767627716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03407, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.9873471856117249, + "learning_rate": 9.985927791195179e-05, + "loss": 0.046224698424339294, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04731, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 1.0582185983657837, + "learning_rate": 9.985347869426513e-05, + "loss": 0.01951594091951847, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01971, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.6863466501235962, + "learning_rate": 9.984756257776302e-05, + "loss": 0.021039601415395737, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02126, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4057074785232544, + "learning_rate": 9.984152957786426e-05, + "loss": 0.008229494094848633, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00826, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.4019395411014557, + "learning_rate": 9.983537971029228e-05, + "loss": 0.010984211228787899, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01104, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.48016035556793213, + "learning_rate": 9.982911299107514e-05, + "loss": 0.012419218197464943, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.440166175365448, + "learning_rate": 9.982272943654535e-05, + "loss": 0.009114887565374374, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00916, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.20877902209758759, + "learning_rate": 9.981622906333999e-05, + "loss": 0.00270082987844944, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0027, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.6577605605125427, + "learning_rate": 9.980961188840055e-05, + "loss": 0.004442816600203514, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00445, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 0.13625678420066833, + "learning_rate": 9.9802877928973e-05, + "loss": 0.0019005294889211655, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0019, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.019028058275580406, + "learning_rate": 9.979602720260762e-05, + "loss": 0.0002803217794280499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00028, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.0662836953997612, + "learning_rate": 9.978905972715901e-05, + "loss": 0.0006815034430474043, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00068, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.9524247646331787, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00464291637763381, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00465, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 0.04438980668783188, + "learning_rate": 9.977477460195198e-05, + "loss": 0.00037421027082018554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 0.7781257629394531, + "learning_rate": 9.976745698942397e-05, + "loss": 0.0023766113445162773, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00238, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.4958593249320984, + "learning_rate": 9.976002270227351e-05, + "loss": 0.029030824080109596, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02946, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 2.34055495262146, + "learning_rate": 9.975247175987611e-05, + "loss": 0.04764559492468834, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0488, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.025128737092018127, + "learning_rate": 9.974480418191133e-05, + "loss": 0.0001649868063395843, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 2.489701986312866, + "learning_rate": 9.973701998836269e-05, + "loss": 0.17427006363868713, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.19038, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 1.9490054845809937, + "learning_rate": 9.972911919951763e-05, + "loss": 0.06718016415834427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06949, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 1.13898766040802, + "learning_rate": 9.972110183596752e-05, + "loss": 0.024798311293125153, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02511, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.4100293517112732, + "learning_rate": 9.971296791860748e-05, + "loss": 0.012426480650901794, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.5648359060287476, + "learning_rate": 9.970471746863644e-05, + "loss": 0.03443408012390137, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03503, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.3096520006656647, + "learning_rate": 9.969635050755704e-05, + "loss": 0.012037754990160465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.01211, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.42714670300483704, + "learning_rate": 9.968786705717555e-05, + "loss": 0.01219131425023079, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01227, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.5599139928817749, + "learning_rate": 9.967926713960188e-05, + "loss": 0.012130084447562695, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0122, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.6217107772827148, + "learning_rate": 9.967055077724944e-05, + "loss": 0.009348927065730095, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00939, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.0592292957007885, + "learning_rate": 9.966171799283513e-05, + "loss": 0.001284315949305892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00129, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 0.462398886680603, + "learning_rate": 9.965276880937933e-05, + "loss": 0.01244058832526207, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01252, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 8.241695404052734, + "learning_rate": 9.964370325020568e-05, + "loss": 0.05807769298553467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0598, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 1.0815411806106567, + "learning_rate": 9.96345213389412e-05, + "loss": 0.017220184206962585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01737, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 1.6311028003692627, + "learning_rate": 9.962522309951614e-05, + "loss": 0.005533762276172638, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00555, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 3.626565456390381, + "learning_rate": 9.961580855616392e-05, + "loss": 0.09052321314811707, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.09475, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 3.3724992275238037, + "learning_rate": 9.960627773342105e-05, + "loss": 0.00758603448048234, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00761, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 0.3157868981361389, + "learning_rate": 9.959663065612716e-05, + "loss": 0.003882938763126731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00389, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.03412550315260887, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0005760709755122662, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00058, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.52850741147995, + "learning_rate": 9.957698783875946e-05, + "loss": 0.010527489706873894, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01058, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.3949584364891052, + "learning_rate": 9.95669921498795e-05, + "loss": 0.007605592254549265, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00763, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 0.018618274480104446, + "learning_rate": 9.955688030883605e-05, + "loss": 0.0003067223879043013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00031, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.11938109248876572, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0008339773630723357, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00083, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 1.4441187381744385, + "learning_rate": 9.953630827597677e-05, + "loss": 0.015039840713143349, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01515, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.04352337867021561, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0008668563095852733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00087, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 10.194028854370117, + "learning_rate": 9.951527195464393e-05, + "loss": 0.01512847188860178, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01524, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.9922191500663757, + "learning_rate": 9.950457975414293e-05, + "loss": 0.012203281745314598, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01228, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 1.3960785865783691, + "learning_rate": 9.949377156413995e-05, + "loss": 0.010049878619611263, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0101, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.6242722272872925, + "learning_rate": 9.948284741280371e-05, + "loss": 0.020798590034246445, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02102, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.9250545501708984, + "learning_rate": 9.947180732860518e-05, + "loss": 0.02078751102089882, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02101, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.44395211338996887, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0027410993352532387, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00274, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.06538265198469162, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0013557756319642067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00136, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.8934104442596436, + "learning_rate": 9.943799176807672e-05, + "loss": 0.009296886622905731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00934, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.285216748714447, + "learning_rate": 9.942648824318006e-05, + "loss": 0.0030592025723308325, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00306, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.6151894330978394, + "learning_rate": 9.941486893230645e-05, + "loss": 0.00584901450201869, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00587, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.3476015329360962, + "learning_rate": 9.940313386573859e-05, + "loss": 0.005612781271338463, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00563, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.04463421553373337, + "learning_rate": 9.939128307406084e-05, + "loss": 0.0004844452196266502, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00048, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.14716975390911102, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0013532615266740322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00135, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.013975728303194046, + "learning_rate": 9.936723443922109e-05, + "loss": 0.00017458554066251963, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.23583655059337616, + "learning_rate": 9.935503665873555e-05, + "loss": 0.0013064221711829305, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00131, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 0.19946330785751343, + "learning_rate": 9.934272327849287e-05, + "loss": 0.0016659648390486836, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00167, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 2.5991828441619873, + "learning_rate": 9.933029433058463e-05, + "loss": 0.03332412242889404, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03389, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.022260449826717377, + "learning_rate": 9.931774984740362e-05, + "loss": 0.00019467048696242273, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00019, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 1.043959140777588, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0496981106698513, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05095, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.009332562796771526, + "learning_rate": 9.929231440629993e-05, + "loss": 0.0001336124842055142, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00013, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.028200708329677582, + "learning_rate": 9.927942351466806e-05, + "loss": 0.00035777001176029444, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00036, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.06076425686478615, + "learning_rate": 9.926641722034484e-05, + "loss": 0.001383051392622292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00138, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.12561249732971191, + "learning_rate": 9.925329555722777e-05, + "loss": 0.0026724478229880333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00268, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 0.5528528094291687, + "learning_rate": 9.924005855951503e-05, + "loss": 0.01256745308637619, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01265, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 2.376209259033203, + "learning_rate": 9.922670626170537e-05, + "loss": 0.017799941822886467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01796, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.2762843370437622, + "learning_rate": 9.921323869859805e-05, + "loss": 0.007115472108125687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00714, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.24070966243743896, + "learning_rate": 9.919965590529272e-05, + "loss": 0.0015583797357976437, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00156, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.10395435988903046, + "learning_rate": 9.918595791718938e-05, + "loss": 0.0007914248271845281, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00079, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.5730319023132324, + "learning_rate": 9.917214476998825e-05, + "loss": 0.005572602618485689, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00559, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.1355881541967392, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0004524950636550784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00045, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.07415846735239029, + "learning_rate": 9.914417314259398e-05, + "loss": 0.0004993464681319892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0005, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 0.22872294485569, + "learning_rate": 9.913001473530153e-05, + "loss": 0.0009260782971978188, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00093, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.006719402503222227, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001365313510177657, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.020894810557365417, + "learning_rate": 9.910135291802682e-05, + "loss": 0.00035784178180620074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00036, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.06920765340328217, + "learning_rate": 9.908684958274405e-05, + "loss": 0.000399862794438377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0004, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.008416767232120037, + "learning_rate": 9.907223134666332e-05, + "loss": 4.7169265599222854e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00005, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 0.013850455172359943, + "learning_rate": 9.905749824788326e-05, + "loss": 0.00019475985027384013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00019, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.0023701407480984926, + "learning_rate": 9.904265032480177e-05, + "loss": 3.626237958087586e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.0015644921222701669, + "learning_rate": 9.902768761611609e-05, + "loss": 3.6560850276146084e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 0.54908686876297, + "learning_rate": 9.901261016082262e-05, + "loss": 0.02532411366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02565, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.00214971462264657, + "learning_rate": 9.899741799821673e-05, + "loss": 3.0114068067632616e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0071777780540287495, + "learning_rate": 9.898211116789284e-05, + "loss": 5.1465347496559843e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 1.685914158821106, + "learning_rate": 9.896668970974419e-05, + "loss": 0.014849710278213024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01496, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 1.67842698097229, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0018469173228368163, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.005261159967631102, + "learning_rate": 9.893550307103913e-05, + "loss": 8.17667314549908e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.023177370429039, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00030886579770594835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00031, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.3134678304195404, + "learning_rate": 9.890385840722048e-05, + "loss": 0.0008985826279968023, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0009, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.7450703382492065, + "learning_rate": 9.888786441879897e-05, + "loss": 0.017146039754152298, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01729, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 0.008559435606002808, + "learning_rate": 9.887175604818206e-05, + "loss": 0.0001242949510924518, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.00012, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 1.7312885522842407, + "learning_rate": 9.885553333735204e-05, + "loss": 0.008018076419830322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00805, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 1.6031489372253418, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0027930131182074547, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0028, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.7863869071006775, + "learning_rate": 9.882274506447146e-05, + "loss": 0.02333652786910534, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02361, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.6194532513618469, + "learning_rate": 9.88061795878749e-05, + "loss": 0.003525766544044018, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00353, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 1.360031008720398, + "learning_rate": 9.878949994197307e-05, + "loss": 0.017261315137147903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01741, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03367060795426369, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0006716687930747867, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00067, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.1465068757534027, + "learning_rate": 9.875579831643538e-05, + "loss": 0.0024646068923175335, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00247, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.6451879739761353, + "learning_rate": 9.873877642463397e-05, + "loss": 0.016013173386454582, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01614, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.5427649617195129, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010463962331414223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01052, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.14428693056106567, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0027021225541830063, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00271, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.41795286536216736, + "learning_rate": 9.868702696634756e-05, + "loss": 0.014849536120891571, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01496, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.33399561047554016, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0074952589347958565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00752, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.2544633150100708, + "learning_rate": 9.865195795873484e-05, + "loss": 0.005140242632478476, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00515, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.6971493363380432, + "learning_rate": 9.863425278095409e-05, + "loss": 0.02200455032289028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02225, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 82760 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.6525957745810125e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-256/training_args.bin b/aft/coin/checkpoint-256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-32/README.md b/aft/coin/checkpoint-32/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-32/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-32/adapter_config.json b/aft/coin/checkpoint-32/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-32/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-32/adapter_model.safetensors b/aft/coin/checkpoint-32/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..06608c89640cf3deb26a74617418b8216e111242 --- /dev/null +++ b/aft/coin/checkpoint-32/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43fdc1c78c1b7c22eff6d581c8c71d8d7781483598a30ff6ad898be418461ce0 +size 1047628488 diff --git a/aft/coin/checkpoint-32/chat_template.jinja b/aft/coin/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-32/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-32/tokenizer.json b/aft/coin/checkpoint-32/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-32/tokenizer_config.json b/aft/coin/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-32/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-32/tokens_state.json b/aft/coin/checkpoint-32/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa68cfea5824a030d620df18bcfe8c6bc475e0e --- /dev/null +++ b/aft/coin/checkpoint-32/tokens_state.json @@ -0,0 +1 @@ +{"total": 482372, "trainable": 10340} \ No newline at end of file diff --git a/aft/coin/checkpoint-32/trainer_state.json b/aft/coin/checkpoint-32/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8a821af3ffb866b8ba006db24e6f761cc1722b61 --- /dev/null +++ b/aft/coin/checkpoint-32/trainer_state.json @@ -0,0 +1,482 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.310309566151949e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-32/training_args.bin b/aft/coin/checkpoint-32/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-4/README.md b/aft/coin/checkpoint-4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-4/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-4/adapter_config.json b/aft/coin/checkpoint-4/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-4/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-4/adapter_model.safetensors b/aft/coin/checkpoint-4/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fce015428cbb4969be29ac0e97d848298645ccd8 --- /dev/null +++ b/aft/coin/checkpoint-4/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693e370377a516b950d1c23bfd5948c4dd74aef518e85f7892afdf440da786c9 +size 1047628488 diff --git a/aft/coin/checkpoint-4/chat_template.jinja b/aft/coin/checkpoint-4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-4/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-4/tokenizer.json b/aft/coin/checkpoint-4/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-4/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-4/tokenizer_config.json b/aft/coin/checkpoint-4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-4/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-4/tokens_state.json b/aft/coin/checkpoint-4/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1434a7b482c4b4f906f66d29b16e54ca75c60bc8 --- /dev/null +++ b/aft/coin/checkpoint-4/tokens_state.json @@ -0,0 +1 @@ +{"total": 60368, "trainable": 1288} \ No newline at end of file diff --git a/aft/coin/checkpoint-4/trainer_state.json b/aft/coin/checkpoint-4/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7a47903c4658c4ba0328b7960983097fb15bb66a --- /dev/null +++ b/aft/coin/checkpoint-4/trainer_state.json @@ -0,0 +1,90 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.0625, + "eval_steps": 500, + "global_step": 4, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4142793692201472.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-4/training_args.bin b/aft/coin/checkpoint-4/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-4/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-512/README.md b/aft/coin/checkpoint-512/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-512/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-512/adapter_config.json b/aft/coin/checkpoint-512/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-512/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-512/adapter_model.safetensors b/aft/coin/checkpoint-512/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0a2fc9107a5fff3fd9831c249a227517ccd5316 --- /dev/null +++ b/aft/coin/checkpoint-512/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b72670079de9472797a158f36221f77e7967fc15e735264e6f4cc549c9540a55 +size 1047628488 diff --git a/aft/coin/checkpoint-512/chat_template.jinja b/aft/coin/checkpoint-512/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-512/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-512/tokenizer.json b/aft/coin/checkpoint-512/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-512/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-512/tokenizer_config.json b/aft/coin/checkpoint-512/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-512/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-512/tokens_state.json b/aft/coin/checkpoint-512/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a35726e3eba875eb21aa10ca394e288782269677 --- /dev/null +++ b/aft/coin/checkpoint-512/tokens_state.json @@ -0,0 +1 @@ +{"total": 7730080, "trainable": 165520} \ No newline at end of file diff --git a/aft/coin/checkpoint-512/trainer_state.json b/aft/coin/checkpoint-512/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7365a9fe37e466109b79b9df53b7a4bfe2e9dde4 --- /dev/null +++ b/aft/coin/checkpoint-512/trainer_state.json @@ -0,0 +1,7202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0028458833694458, + "learning_rate": 6.274509803921569e-05, + "loss": 0.01862957514822483, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0188, + "step": 65, + "tokens/total": 981308, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 21009 + }, + { + "epoch": 1.03125, + "grad_norm": 0.9001771807670593, + "learning_rate": 6.372549019607843e-05, + "loss": 0.021384725347161293, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02162, + "step": 66, + "tokens/total": 996432, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 21334 + }, + { + "epoch": 1.046875, + "grad_norm": 0.5960713028907776, + "learning_rate": 6.470588235294118e-05, + "loss": 0.010063373483717442, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01011, + "step": 67, + "tokens/total": 1011540, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 21655 + }, + { + "epoch": 1.0625, + "grad_norm": 2.038512706756592, + "learning_rate": 6.568627450980392e-05, + "loss": 0.03831503912806511, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03906, + "step": 68, + "tokens/total": 1026576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 21981 + }, + { + "epoch": 1.078125, + "grad_norm": 2.5726208686828613, + "learning_rate": 6.666666666666667e-05, + "loss": 0.03134162724018097, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03184, + "step": 69, + "tokens/total": 1041696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 22301 + }, + { + "epoch": 1.09375, + "grad_norm": 0.793451189994812, + "learning_rate": 6.764705882352942e-05, + "loss": 0.028099995106458664, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0285, + "step": 70, + "tokens/total": 1056788, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 22627 + }, + { + "epoch": 1.109375, + "grad_norm": 0.7262976169586182, + "learning_rate": 6.862745098039216e-05, + "loss": 0.02038084715604782, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02059, + "step": 71, + "tokens/total": 1071884, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 22948 + }, + { + "epoch": 1.125, + "grad_norm": 0.532096803188324, + "learning_rate": 6.96078431372549e-05, + "loss": 0.013697818852961063, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01379, + "step": 72, + "tokens/total": 1087020, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 23273 + }, + { + "epoch": 1.140625, + "grad_norm": 0.7747638821601868, + "learning_rate": 7.058823529411765e-05, + "loss": 0.023866616189479828, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02415, + "step": 73, + "tokens/total": 1102216, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 23599 + }, + { + "epoch": 1.15625, + "grad_norm": 0.49953746795654297, + "learning_rate": 7.156862745098039e-05, + "loss": 0.014121961779892445, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01422, + "step": 74, + "tokens/total": 1117340, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 23923 + }, + { + "epoch": 1.171875, + "grad_norm": 0.3340316116809845, + "learning_rate": 7.254901960784314e-05, + "loss": 0.012052969075739384, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01213, + "step": 75, + "tokens/total": 1132380, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 24242 + }, + { + "epoch": 1.1875, + "grad_norm": 0.735864520072937, + "learning_rate": 7.352941176470589e-05, + "loss": 0.04014549404382706, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04096, + "step": 76, + "tokens/total": 1147456, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 24562 + }, + { + "epoch": 1.203125, + "grad_norm": 1.4149185419082642, + "learning_rate": 7.450980392156864e-05, + "loss": 0.045495182275772095, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04655, + "step": 77, + "tokens/total": 1162620, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 24881 + }, + { + "epoch": 1.21875, + "grad_norm": 0.31831473112106323, + "learning_rate": 7.549019607843137e-05, + "loss": 0.011568848043680191, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01164, + "step": 78, + "tokens/total": 1177684, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25205 + }, + { + "epoch": 1.234375, + "grad_norm": 0.5228955745697021, + "learning_rate": 7.647058823529411e-05, + "loss": 0.02045898698270321, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02067, + "step": 79, + "tokens/total": 1192896, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 25524 + }, + { + "epoch": 1.25, + "grad_norm": 0.5053377151489258, + "learning_rate": 7.745098039215687e-05, + "loss": 0.020432129502296448, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02064, + "step": 80, + "tokens/total": 1207936, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 25846 + }, + { + "epoch": 1.265625, + "grad_norm": 0.35163524746894836, + "learning_rate": 7.843137254901961e-05, + "loss": 0.01403958909213543, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.01414, + "step": 81, + "tokens/total": 1223060, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 26170 + }, + { + "epoch": 1.28125, + "grad_norm": 0.6905986070632935, + "learning_rate": 7.941176470588235e-05, + "loss": 0.03329383581876755, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.03385, + "step": 82, + "tokens/total": 1238128, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 26495 + }, + { + "epoch": 1.296875, + "grad_norm": 0.4459448456764221, + "learning_rate": 8.039215686274511e-05, + "loss": 0.006232030689716339, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00625, + "step": 83, + "tokens/total": 1253184, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 26814 + }, + { + "epoch": 1.3125, + "grad_norm": 0.2638562321662903, + "learning_rate": 8.137254901960785e-05, + "loss": 0.006893642246723175, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00692, + "step": 84, + "tokens/total": 1268316, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 27139 + }, + { + "epoch": 1.328125, + "grad_norm": 1.170509696006775, + "learning_rate": 8.23529411764706e-05, + "loss": 0.0398201048374176, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.04062, + "step": 85, + "tokens/total": 1283432, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 27461 + }, + { + "epoch": 1.34375, + "grad_norm": 0.8102952837944031, + "learning_rate": 8.333333333333334e-05, + "loss": 0.018316198140382767, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01848, + "step": 86, + "tokens/total": 1298488, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 27785 + }, + { + "epoch": 1.359375, + "grad_norm": 0.6812735795974731, + "learning_rate": 8.431372549019608e-05, + "loss": 0.02392837405204773, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02422, + "step": 87, + "tokens/total": 1313556, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 28106 + }, + { + "epoch": 1.375, + "grad_norm": 1.251705288887024, + "learning_rate": 8.529411764705883e-05, + "loss": 0.024196332320570946, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02449, + "step": 88, + "tokens/total": 1328640, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 28427 + }, + { + "epoch": 1.390625, + "grad_norm": 0.5772960782051086, + "learning_rate": 8.627450980392158e-05, + "loss": 0.021560901775956154, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0218, + "step": 89, + "tokens/total": 1343836, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 28750 + }, + { + "epoch": 1.40625, + "grad_norm": 1.6470917463302612, + "learning_rate": 8.725490196078432e-05, + "loss": 0.04491877555847168, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.04594, + "step": 90, + "tokens/total": 1358884, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 29074 + }, + { + "epoch": 1.421875, + "grad_norm": 0.41440942883491516, + "learning_rate": 8.823529411764706e-05, + "loss": 0.0230612400919199, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02333, + "step": 91, + "tokens/total": 1374012, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 29398 + }, + { + "epoch": 1.4375, + "grad_norm": 0.39720970392227173, + "learning_rate": 8.921568627450981e-05, + "loss": 0.013313441537320614, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0134, + "step": 92, + "tokens/total": 1389152, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 29723 + }, + { + "epoch": 1.453125, + "grad_norm": 0.8898259401321411, + "learning_rate": 9.019607843137255e-05, + "loss": 0.015421045944094658, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.01554, + "step": 93, + "tokens/total": 1404108, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 30044 + }, + { + "epoch": 1.46875, + "grad_norm": 8.74992847442627, + "learning_rate": 9.11764705882353e-05, + "loss": 0.03271842002868652, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03326, + "step": 94, + "tokens/total": 1419316, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 30370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.18793171644210815, + "learning_rate": 9.215686274509804e-05, + "loss": 0.0015747153665870428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00158, + "step": 95, + "tokens/total": 1434464, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 30692 + }, + { + "epoch": 1.5, + "grad_norm": 0.9997888207435608, + "learning_rate": 9.313725490196079e-05, + "loss": 0.03334064409136772, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0339, + "step": 96, + "tokens/total": 1449528, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 31020 + }, + { + "epoch": 1.515625, + "grad_norm": 0.6498851776123047, + "learning_rate": 9.411764705882353e-05, + "loss": 0.023218588903546333, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02349, + "step": 97, + "tokens/total": 1464644, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 31342 + }, + { + "epoch": 1.53125, + "grad_norm": 0.5593264102935791, + "learning_rate": 9.509803921568627e-05, + "loss": 0.005530651658773422, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00555, + "step": 98, + "tokens/total": 1479792, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 31665 + }, + { + "epoch": 1.546875, + "grad_norm": 0.8789476752281189, + "learning_rate": 9.607843137254903e-05, + "loss": 0.04554657265543938, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0466, + "step": 99, + "tokens/total": 1494976, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 31990 + }, + { + "epoch": 1.5625, + "grad_norm": 0.5140370726585388, + "learning_rate": 9.705882352941177e-05, + "loss": 0.014101475477218628, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0142, + "step": 100, + "tokens/total": 1510128, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 32314 + }, + { + "epoch": 1.578125, + "grad_norm": 0.5285096764564514, + "learning_rate": 9.80392156862745e-05, + "loss": 0.030611136928200722, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03108, + "step": 101, + "tokens/total": 1525260, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 32639 + }, + { + "epoch": 1.59375, + "grad_norm": 0.6092981696128845, + "learning_rate": 9.901960784313727e-05, + "loss": 0.028998689725995064, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02942, + "step": 102, + "tokens/total": 1540384, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 32962 + }, + { + "epoch": 1.609375, + "grad_norm": 1.0235849618911743, + "learning_rate": 0.0001, + "loss": 0.025761116296052933, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0261, + "step": 103, + "tokens/total": 1555516, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33289 + }, + { + "epoch": 1.625, + "grad_norm": 0.9383920431137085, + "learning_rate": 9.999994135965763e-05, + "loss": 0.02419213019311428, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02449, + "step": 104, + "tokens/total": 1570608, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 33611 + }, + { + "epoch": 1.640625, + "grad_norm": 0.8995917439460754, + "learning_rate": 9.999976543878332e-05, + "loss": 0.011181461624801159, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01124, + "step": 105, + "tokens/total": 1585692, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 33936 + }, + { + "epoch": 1.65625, + "grad_norm": 3.9120519161224365, + "learning_rate": 9.999947223783559e-05, + "loss": 0.10826049745082855, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.11434, + "step": 106, + "tokens/total": 1600828, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 34260 + }, + { + "epoch": 1.671875, + "grad_norm": 1.801079511642456, + "learning_rate": 9.999906175757857e-05, + "loss": 0.029263045638799667, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0297, + "step": 107, + "tokens/total": 1615928, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 34586 + }, + { + "epoch": 1.6875, + "grad_norm": 3.040914535522461, + "learning_rate": 9.99985339990821e-05, + "loss": 0.04611361399292946, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04719, + "step": 108, + "tokens/total": 1631080, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 34909 + }, + { + "epoch": 1.703125, + "grad_norm": 0.6034630537033081, + "learning_rate": 9.999788896372159e-05, + "loss": 0.0116285290569067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0117, + "step": 109, + "tokens/total": 1646176, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 35231 + }, + { + "epoch": 1.71875, + "grad_norm": 1.238577961921692, + "learning_rate": 9.999712665317819e-05, + "loss": 0.037302885204553604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.03801, + "step": 110, + "tokens/total": 1661304, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 35558 + }, + { + "epoch": 1.734375, + "grad_norm": 0.404407262802124, + "learning_rate": 9.999624706943868e-05, + "loss": 0.010177684016525745, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01023, + "step": 111, + "tokens/total": 1676396, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 35884 + }, + { + "epoch": 1.75, + "grad_norm": 1.1107022762298584, + "learning_rate": 9.999525021479544e-05, + "loss": 0.016156412661075592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01629, + "step": 112, + "tokens/total": 1691460, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 36210 + }, + { + "epoch": 1.765625, + "grad_norm": 2.0084729194641113, + "learning_rate": 9.99941360918465e-05, + "loss": 0.022498758509755135, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02275, + "step": 113, + "tokens/total": 1706496, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 36533 + }, + { + "epoch": 1.78125, + "grad_norm": 0.9237216711044312, + "learning_rate": 9.999290470349554e-05, + "loss": 0.009256366640329361, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0093, + "step": 114, + "tokens/total": 1721644, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 36854 + }, + { + "epoch": 1.796875, + "grad_norm": 1.0012309551239014, + "learning_rate": 9.999155605295188e-05, + "loss": 0.01096038706600666, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.01102, + "step": 115, + "tokens/total": 1736692, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 37176 + }, + { + "epoch": 1.8125, + "grad_norm": 0.36062437295913696, + "learning_rate": 9.999009014373037e-05, + "loss": 0.0026956750079989433, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0027, + "step": 116, + "tokens/total": 1751844, + "tokens/train_per_sec_per_gpu": 8.78, + "tokens/trainable": 37498 + }, + { + "epoch": 1.828125, + "grad_norm": 0.09146630764007568, + "learning_rate": 9.998850697965153e-05, + "loss": 0.0006923460750840604, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00069, + "step": 117, + "tokens/total": 1766996, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 37825 + }, + { + "epoch": 1.84375, + "grad_norm": 0.35573962330818176, + "learning_rate": 9.99868065648415e-05, + "loss": 0.0032197514083236456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00322, + "step": 118, + "tokens/total": 1781996, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 38145 + }, + { + "epoch": 1.859375, + "grad_norm": 0.18824201822280884, + "learning_rate": 9.998498890373194e-05, + "loss": 0.0015725638950243592, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00157, + "step": 119, + "tokens/total": 1797092, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 38469 + }, + { + "epoch": 1.875, + "grad_norm": 2.2947869300842285, + "learning_rate": 9.998305400106009e-05, + "loss": 0.09606514871120453, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.10083, + "step": 120, + "tokens/total": 1812120, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 38793 + }, + { + "epoch": 1.890625, + "grad_norm": 0.6851844787597656, + "learning_rate": 9.998100186186879e-05, + "loss": 0.006874916143715382, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0069, + "step": 121, + "tokens/total": 1827244, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 39119 + }, + { + "epoch": 1.90625, + "grad_norm": 1.7458359003067017, + "learning_rate": 9.997883249150642e-05, + "loss": 0.020516997203230858, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02073, + "step": 122, + "tokens/total": 1842452, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 39445 + }, + { + "epoch": 1.921875, + "grad_norm": 0.7431558966636658, + "learning_rate": 9.99765458956268e-05, + "loss": 0.014651929959654808, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01476, + "step": 123, + "tokens/total": 1857472, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 39765 + }, + { + "epoch": 1.9375, + "grad_norm": 0.16357871890068054, + "learning_rate": 9.997414208018943e-05, + "loss": 0.003089388832449913, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00309, + "step": 124, + "tokens/total": 1872560, + "tokens/train_per_sec_per_gpu": 9.59, + "tokens/trainable": 40088 + }, + { + "epoch": 1.953125, + "grad_norm": 0.36700522899627686, + "learning_rate": 9.997162105145916e-05, + "loss": 0.01339892391115427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01349, + "step": 125, + "tokens/total": 1887756, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 40416 + }, + { + "epoch": 1.96875, + "grad_norm": 0.22755062580108643, + "learning_rate": 9.996898281600642e-05, + "loss": 0.005444459617137909, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00546, + "step": 126, + "tokens/total": 1902868, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 40738 + }, + { + "epoch": 1.984375, + "grad_norm": 0.339722216129303, + "learning_rate": 9.996622738070706e-05, + "loss": 0.004514522384852171, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00452, + "step": 127, + "tokens/total": 1917972, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41057 + }, + { + "epoch": 2.0, + "grad_norm": 1.7278125286102295, + "learning_rate": 9.996335475274243e-05, + "loss": 0.024016786366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02431, + "step": 128, + "tokens/total": 1932952, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 41380 + }, + { + "epoch": 2.015625, + "grad_norm": 0.47769469022750854, + "learning_rate": 9.996036493959926e-05, + "loss": 0.010395423509180546, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01045, + "step": 129, + "tokens/total": 1948040, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 41705 + }, + { + "epoch": 2.03125, + "grad_norm": 0.4170791804790497, + "learning_rate": 9.99572579490697e-05, + "loss": 0.004119648598134518, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00413, + "step": 130, + "tokens/total": 1963040, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 42027 + }, + { + "epoch": 2.046875, + "grad_norm": 0.37145885825157166, + "learning_rate": 9.995403378925132e-05, + "loss": 0.00920130219310522, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00924, + "step": 131, + "tokens/total": 1978112, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 42349 + }, + { + "epoch": 2.0625, + "grad_norm": 0.06787586212158203, + "learning_rate": 9.995069246854705e-05, + "loss": 0.0012981402687728405, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0013, + "step": 132, + "tokens/total": 1993264, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 42674 + }, + { + "epoch": 2.078125, + "grad_norm": 0.7351065874099731, + "learning_rate": 9.994723399566515e-05, + "loss": 0.021887682378292084, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02213, + "step": 133, + "tokens/total": 2008392, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 42995 + }, + { + "epoch": 2.09375, + "grad_norm": 0.590771496295929, + "learning_rate": 9.994365837961926e-05, + "loss": 0.009705535136163235, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00975, + "step": 134, + "tokens/total": 2023500, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 43319 + }, + { + "epoch": 2.109375, + "grad_norm": 0.3608993589878082, + "learning_rate": 9.993996562972823e-05, + "loss": 0.008060690015554428, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00809, + "step": 135, + "tokens/total": 2038512, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 43643 + }, + { + "epoch": 2.125, + "grad_norm": 0.48873215913772583, + "learning_rate": 9.993615575561626e-05, + "loss": 0.008283237926661968, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00832, + "step": 136, + "tokens/total": 2053608, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 43967 + }, + { + "epoch": 2.140625, + "grad_norm": 0.7132001519203186, + "learning_rate": 9.99322287672128e-05, + "loss": 0.024729935452342033, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.02504, + "step": 137, + "tokens/total": 2068668, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 44293 + }, + { + "epoch": 2.15625, + "grad_norm": 0.09246033430099487, + "learning_rate": 9.992818467475254e-05, + "loss": 0.001992235891520977, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00199, + "step": 138, + "tokens/total": 2083720, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 44615 + }, + { + "epoch": 2.171875, + "grad_norm": 0.7287611365318298, + "learning_rate": 9.992402348877528e-05, + "loss": 0.021320564672350883, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02155, + "step": 139, + "tokens/total": 2098820, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 44941 + }, + { + "epoch": 2.1875, + "grad_norm": 0.2133086621761322, + "learning_rate": 9.991974522012612e-05, + "loss": 0.00337275885976851, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00338, + "step": 140, + "tokens/total": 2113952, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 45264 + }, + { + "epoch": 2.203125, + "grad_norm": 0.37355223298072815, + "learning_rate": 9.99153498799552e-05, + "loss": 0.007823438383638859, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00785, + "step": 141, + "tokens/total": 2129076, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 45586 + }, + { + "epoch": 2.21875, + "grad_norm": 0.06703885644674301, + "learning_rate": 9.991083747971783e-05, + "loss": 0.0005649570957757533, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00057, + "step": 142, + "tokens/total": 2144200, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 45908 + }, + { + "epoch": 2.234375, + "grad_norm": 0.06440247595310211, + "learning_rate": 9.990620803117441e-05, + "loss": 0.0006133198039606214, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00061, + "step": 143, + "tokens/total": 2159344, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 46228 + }, + { + "epoch": 2.25, + "grad_norm": 0.34707385301589966, + "learning_rate": 9.990146154639039e-05, + "loss": 0.0035533613990992308, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00356, + "step": 144, + "tokens/total": 2174344, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 46556 + }, + { + "epoch": 2.265625, + "grad_norm": 2.017024517059326, + "learning_rate": 9.98965980377362e-05, + "loss": 0.042595766484737396, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04352, + "step": 145, + "tokens/total": 2189384, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 46877 + }, + { + "epoch": 2.28125, + "grad_norm": 0.785468339920044, + "learning_rate": 9.989161751788734e-05, + "loss": 0.002102769911289215, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0021, + "step": 146, + "tokens/total": 2204604, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 47205 + }, + { + "epoch": 2.296875, + "grad_norm": 2.8202157020568848, + "learning_rate": 9.98865199998242e-05, + "loss": 0.01083361729979515, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01089, + "step": 147, + "tokens/total": 2219772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 47526 + }, + { + "epoch": 2.3125, + "grad_norm": 1.0493052005767822, + "learning_rate": 9.988130549683211e-05, + "loss": 0.014770902693271637, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.01488, + "step": 148, + "tokens/total": 2234872, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 47851 + }, + { + "epoch": 2.328125, + "grad_norm": 2.157977819442749, + "learning_rate": 9.987597402250136e-05, + "loss": 0.009032780304551125, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00907, + "step": 149, + "tokens/total": 2250024, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 48174 + }, + { + "epoch": 2.34375, + "grad_norm": 0.029030824080109596, + "learning_rate": 9.987052559072696e-05, + "loss": 0.00013317995762918144, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 150, + "tokens/total": 2265020, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 48499 + }, + { + "epoch": 2.359375, + "grad_norm": 2.6754238605499268, + "learning_rate": 9.98649602157089e-05, + "loss": 0.033499784767627716, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03407, + "step": 151, + "tokens/total": 2280188, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 48824 + }, + { + "epoch": 2.375, + "grad_norm": 0.9873471856117249, + "learning_rate": 9.985927791195179e-05, + "loss": 0.046224698424339294, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04731, + "step": 152, + "tokens/total": 2295256, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 49147 + }, + { + "epoch": 2.390625, + "grad_norm": 1.0582185983657837, + "learning_rate": 9.985347869426513e-05, + "loss": 0.01951594091951847, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01971, + "step": 153, + "tokens/total": 2310476, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 49470 + }, + { + "epoch": 2.40625, + "grad_norm": 0.6863466501235962, + "learning_rate": 9.984756257776302e-05, + "loss": 0.021039601415395737, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02126, + "step": 154, + "tokens/total": 2325512, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 49791 + }, + { + "epoch": 2.421875, + "grad_norm": 0.4057074785232544, + "learning_rate": 9.984152957786426e-05, + "loss": 0.008229494094848633, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00826, + "step": 155, + "tokens/total": 2340568, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 50116 + }, + { + "epoch": 2.4375, + "grad_norm": 0.4019395411014557, + "learning_rate": 9.983537971029228e-05, + "loss": 0.010984211228787899, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01104, + "step": 156, + "tokens/total": 2355752, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 50444 + }, + { + "epoch": 2.453125, + "grad_norm": 0.48016035556793213, + "learning_rate": 9.982911299107514e-05, + "loss": 0.012419218197464943, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 157, + "tokens/total": 2370824, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 50768 + }, + { + "epoch": 2.46875, + "grad_norm": 0.440166175365448, + "learning_rate": 9.982272943654535e-05, + "loss": 0.009114887565374374, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00916, + "step": 158, + "tokens/total": 2385908, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 51091 + }, + { + "epoch": 2.484375, + "grad_norm": 0.20877902209758759, + "learning_rate": 9.981622906333999e-05, + "loss": 0.00270082987844944, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0027, + "step": 159, + "tokens/total": 2401084, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 51415 + }, + { + "epoch": 2.5, + "grad_norm": 0.6577605605125427, + "learning_rate": 9.980961188840055e-05, + "loss": 0.004442816600203514, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00445, + "step": 160, + "tokens/total": 2416224, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 51739 + }, + { + "epoch": 2.515625, + "grad_norm": 0.13625678420066833, + "learning_rate": 9.9802877928973e-05, + "loss": 0.0019005294889211655, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0019, + "step": 161, + "tokens/total": 2431356, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 52059 + }, + { + "epoch": 2.53125, + "grad_norm": 0.019028058275580406, + "learning_rate": 9.979602720260762e-05, + "loss": 0.0002803217794280499, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00028, + "step": 162, + "tokens/total": 2446500, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 52378 + }, + { + "epoch": 2.546875, + "grad_norm": 0.0662836953997612, + "learning_rate": 9.978905972715901e-05, + "loss": 0.0006815034430474043, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00068, + "step": 163, + "tokens/total": 2461480, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 52699 + }, + { + "epoch": 2.5625, + "grad_norm": 0.9524247646331787, + "learning_rate": 9.97819755207861e-05, + "loss": 0.00464291637763381, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00465, + "step": 164, + "tokens/total": 2476532, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 53021 + }, + { + "epoch": 2.578125, + "grad_norm": 0.04438980668783188, + "learning_rate": 9.977477460195198e-05, + "loss": 0.00037421027082018554, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00037, + "step": 165, + "tokens/total": 2491592, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 53346 + }, + { + "epoch": 2.59375, + "grad_norm": 0.7781257629394531, + "learning_rate": 9.976745698942397e-05, + "loss": 0.0023766113445162773, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00238, + "step": 166, + "tokens/total": 2506632, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 53665 + }, + { + "epoch": 2.609375, + "grad_norm": 0.4958593249320984, + "learning_rate": 9.976002270227351e-05, + "loss": 0.029030824080109596, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02946, + "step": 167, + "tokens/total": 2521780, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 53990 + }, + { + "epoch": 2.625, + "grad_norm": 2.34055495262146, + "learning_rate": 9.975247175987611e-05, + "loss": 0.04764559492468834, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0488, + "step": 168, + "tokens/total": 2536820, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 54311 + }, + { + "epoch": 2.640625, + "grad_norm": 0.025128737092018127, + "learning_rate": 9.974480418191133e-05, + "loss": 0.0001649868063395843, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 169, + "tokens/total": 2551952, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 54635 + }, + { + "epoch": 2.65625, + "grad_norm": 2.489701986312866, + "learning_rate": 9.973701998836269e-05, + "loss": 0.17427006363868713, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.19038, + "step": 170, + "tokens/total": 2566988, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 54960 + }, + { + "epoch": 2.671875, + "grad_norm": 1.9490054845809937, + "learning_rate": 9.972911919951763e-05, + "loss": 0.06718016415834427, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06949, + "step": 171, + "tokens/total": 2582044, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 55277 + }, + { + "epoch": 2.6875, + "grad_norm": 1.13898766040802, + "learning_rate": 9.972110183596752e-05, + "loss": 0.024798311293125153, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02511, + "step": 172, + "tokens/total": 2597156, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55598 + }, + { + "epoch": 2.703125, + "grad_norm": 0.4100293517112732, + "learning_rate": 9.971296791860748e-05, + "loss": 0.012426480650901794, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0125, + "step": 173, + "tokens/total": 2612288, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 55923 + }, + { + "epoch": 2.71875, + "grad_norm": 0.5648359060287476, + "learning_rate": 9.970471746863644e-05, + "loss": 0.03443408012390137, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.03503, + "step": 174, + "tokens/total": 2627364, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 56252 + }, + { + "epoch": 2.734375, + "grad_norm": 0.3096520006656647, + "learning_rate": 9.969635050755704e-05, + "loss": 0.012037754990160465, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.01211, + "step": 175, + "tokens/total": 2642320, + "tokens/train_per_sec_per_gpu": 9.86, + "tokens/trainable": 56578 + }, + { + "epoch": 2.75, + "grad_norm": 0.42714670300483704, + "learning_rate": 9.968786705717555e-05, + "loss": 0.01219131425023079, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01227, + "step": 176, + "tokens/total": 2657376, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 56901 + }, + { + "epoch": 2.765625, + "grad_norm": 0.5599139928817749, + "learning_rate": 9.967926713960188e-05, + "loss": 0.012130084447562695, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0122, + "step": 177, + "tokens/total": 2672512, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 57221 + }, + { + "epoch": 2.78125, + "grad_norm": 0.6217107772827148, + "learning_rate": 9.967055077724944e-05, + "loss": 0.009348927065730095, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00939, + "step": 178, + "tokens/total": 2687576, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 57541 + }, + { + "epoch": 2.796875, + "grad_norm": 0.0592292957007885, + "learning_rate": 9.966171799283513e-05, + "loss": 0.001284315949305892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00129, + "step": 179, + "tokens/total": 2702656, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 57863 + }, + { + "epoch": 2.8125, + "grad_norm": 0.462398886680603, + "learning_rate": 9.965276880937933e-05, + "loss": 0.01244058832526207, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01252, + "step": 180, + "tokens/total": 2717772, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 58185 + }, + { + "epoch": 2.828125, + "grad_norm": 8.241695404052734, + "learning_rate": 9.964370325020568e-05, + "loss": 0.05807769298553467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0598, + "step": 181, + "tokens/total": 2732916, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 58511 + }, + { + "epoch": 2.84375, + "grad_norm": 1.0815411806106567, + "learning_rate": 9.96345213389412e-05, + "loss": 0.017220184206962585, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01737, + "step": 182, + "tokens/total": 2748052, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 58834 + }, + { + "epoch": 2.859375, + "grad_norm": 1.6311028003692627, + "learning_rate": 9.962522309951614e-05, + "loss": 0.005533762276172638, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00555, + "step": 183, + "tokens/total": 2763144, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 59154 + }, + { + "epoch": 2.875, + "grad_norm": 3.626565456390381, + "learning_rate": 9.961580855616392e-05, + "loss": 0.09052321314811707, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.09475, + "step": 184, + "tokens/total": 2778300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 59481 + }, + { + "epoch": 2.890625, + "grad_norm": 3.3724992275238037, + "learning_rate": 9.960627773342105e-05, + "loss": 0.00758603448048234, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00761, + "step": 185, + "tokens/total": 2793308, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 59807 + }, + { + "epoch": 2.90625, + "grad_norm": 0.3157868981361389, + "learning_rate": 9.959663065612716e-05, + "loss": 0.003882938763126731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00389, + "step": 186, + "tokens/total": 2808348, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 60132 + }, + { + "epoch": 2.921875, + "grad_norm": 0.03412550315260887, + "learning_rate": 9.958686734942479e-05, + "loss": 0.0005760709755122662, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00058, + "step": 187, + "tokens/total": 2823388, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 60456 + }, + { + "epoch": 2.9375, + "grad_norm": 0.52850741147995, + "learning_rate": 9.957698783875946e-05, + "loss": 0.010527489706873894, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.01058, + "step": 188, + "tokens/total": 2838480, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 60779 + }, + { + "epoch": 2.953125, + "grad_norm": 0.3949584364891052, + "learning_rate": 9.95669921498795e-05, + "loss": 0.007605592254549265, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00763, + "step": 189, + "tokens/total": 2853540, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 61098 + }, + { + "epoch": 2.96875, + "grad_norm": 0.018618274480104446, + "learning_rate": 9.955688030883605e-05, + "loss": 0.0003067223879043013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00031, + "step": 190, + "tokens/total": 2868556, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 61426 + }, + { + "epoch": 2.984375, + "grad_norm": 0.11938109248876572, + "learning_rate": 9.954665234198296e-05, + "loss": 0.0008339773630723357, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00083, + "step": 191, + "tokens/total": 2883668, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 61747 + }, + { + "epoch": 3.0, + "grad_norm": 1.4441187381744385, + "learning_rate": 9.953630827597677e-05, + "loss": 0.015039840713143349, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01515, + "step": 192, + "tokens/total": 2898788, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 62070 + }, + { + "epoch": 3.015625, + "grad_norm": 0.04352337867021561, + "learning_rate": 9.952584813777656e-05, + "loss": 0.0008668563095852733, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00087, + "step": 193, + "tokens/total": 2913816, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 62388 + }, + { + "epoch": 3.03125, + "grad_norm": 10.194028854370117, + "learning_rate": 9.951527195464393e-05, + "loss": 0.01512847188860178, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.01524, + "step": 194, + "tokens/total": 2928936, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 62712 + }, + { + "epoch": 3.046875, + "grad_norm": 0.9922191500663757, + "learning_rate": 9.950457975414293e-05, + "loss": 0.012203281745314598, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01228, + "step": 195, + "tokens/total": 2944008, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 63038 + }, + { + "epoch": 3.0625, + "grad_norm": 1.3960785865783691, + "learning_rate": 9.949377156413995e-05, + "loss": 0.010049878619611263, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0101, + "step": 196, + "tokens/total": 2959084, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 63360 + }, + { + "epoch": 3.078125, + "grad_norm": 0.6242722272872925, + "learning_rate": 9.948284741280371e-05, + "loss": 0.020798590034246445, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02102, + "step": 197, + "tokens/total": 2974084, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 63680 + }, + { + "epoch": 3.09375, + "grad_norm": 0.9250545501708984, + "learning_rate": 9.947180732860518e-05, + "loss": 0.02078751102089882, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02101, + "step": 198, + "tokens/total": 2989244, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 64004 + }, + { + "epoch": 3.109375, + "grad_norm": 0.44395211338996887, + "learning_rate": 9.946065134031739e-05, + "loss": 0.0027410993352532387, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00274, + "step": 199, + "tokens/total": 3004404, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 64331 + }, + { + "epoch": 3.125, + "grad_norm": 0.06538265198469162, + "learning_rate": 9.944937947701553e-05, + "loss": 0.0013557756319642067, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00136, + "step": 200, + "tokens/total": 3019472, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 64653 + }, + { + "epoch": 3.140625, + "grad_norm": 0.8934104442596436, + "learning_rate": 9.943799176807672e-05, + "loss": 0.009296886622905731, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00934, + "step": 201, + "tokens/total": 3034500, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 64971 + }, + { + "epoch": 3.15625, + "grad_norm": 0.285216748714447, + "learning_rate": 9.942648824318006e-05, + "loss": 0.0030592025723308325, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00306, + "step": 202, + "tokens/total": 3049528, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 65295 + }, + { + "epoch": 3.171875, + "grad_norm": 0.6151894330978394, + "learning_rate": 9.941486893230645e-05, + "loss": 0.00584901450201869, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00587, + "step": 203, + "tokens/total": 3064620, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 65615 + }, + { + "epoch": 3.1875, + "grad_norm": 0.3476015329360962, + "learning_rate": 9.940313386573859e-05, + "loss": 0.005612781271338463, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00563, + "step": 204, + "tokens/total": 3079696, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 65936 + }, + { + "epoch": 3.203125, + "grad_norm": 0.04463421553373337, + "learning_rate": 9.939128307406084e-05, + "loss": 0.0004844452196266502, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00048, + "step": 205, + "tokens/total": 3094908, + "tokens/train_per_sec_per_gpu": 9.69, + "tokens/trainable": 66261 + }, + { + "epoch": 3.21875, + "grad_norm": 0.14716975390911102, + "learning_rate": 9.937931658815917e-05, + "loss": 0.0013532615266740322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00135, + "step": 206, + "tokens/total": 3110100, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 66581 + }, + { + "epoch": 3.234375, + "grad_norm": 0.013975728303194046, + "learning_rate": 9.936723443922109e-05, + "loss": 0.00017458554066251963, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00017, + "step": 207, + "tokens/total": 3125184, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 66900 + }, + { + "epoch": 3.25, + "grad_norm": 0.23583655059337616, + "learning_rate": 9.935503665873555e-05, + "loss": 0.0013064221711829305, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00131, + "step": 208, + "tokens/total": 3140340, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 67222 + }, + { + "epoch": 3.265625, + "grad_norm": 0.19946330785751343, + "learning_rate": 9.934272327849287e-05, + "loss": 0.0016659648390486836, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00167, + "step": 209, + "tokens/total": 3155460, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 67547 + }, + { + "epoch": 3.28125, + "grad_norm": 2.5991828441619873, + "learning_rate": 9.933029433058463e-05, + "loss": 0.03332412242889404, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03389, + "step": 210, + "tokens/total": 3170580, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 67872 + }, + { + "epoch": 3.296875, + "grad_norm": 0.022260449826717377, + "learning_rate": 9.931774984740362e-05, + "loss": 0.00019467048696242273, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00019, + "step": 211, + "tokens/total": 3185608, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 68195 + }, + { + "epoch": 3.3125, + "grad_norm": 1.043959140777588, + "learning_rate": 9.930508986164375e-05, + "loss": 0.0496981106698513, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05095, + "step": 212, + "tokens/total": 3200624, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 68515 + }, + { + "epoch": 3.328125, + "grad_norm": 0.009332562796771526, + "learning_rate": 9.929231440629993e-05, + "loss": 0.0001336124842055142, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00013, + "step": 213, + "tokens/total": 3215780, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 68838 + }, + { + "epoch": 3.34375, + "grad_norm": 0.028200708329677582, + "learning_rate": 9.927942351466806e-05, + "loss": 0.00035777001176029444, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00036, + "step": 214, + "tokens/total": 3230876, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 69159 + }, + { + "epoch": 3.359375, + "grad_norm": 0.06076425686478615, + "learning_rate": 9.926641722034484e-05, + "loss": 0.001383051392622292, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00138, + "step": 215, + "tokens/total": 3245960, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 69486 + }, + { + "epoch": 3.375, + "grad_norm": 0.12561249732971191, + "learning_rate": 9.925329555722777e-05, + "loss": 0.0026724478229880333, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00268, + "step": 216, + "tokens/total": 3260956, + "tokens/train_per_sec_per_gpu": 9.15, + "tokens/trainable": 69804 + }, + { + "epoch": 3.390625, + "grad_norm": 0.5528528094291687, + "learning_rate": 9.924005855951503e-05, + "loss": 0.01256745308637619, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01265, + "step": 217, + "tokens/total": 3276044, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 70127 + }, + { + "epoch": 3.40625, + "grad_norm": 2.376209259033203, + "learning_rate": 9.922670626170537e-05, + "loss": 0.017799941822886467, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.01796, + "step": 218, + "tokens/total": 3291192, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 70448 + }, + { + "epoch": 3.421875, + "grad_norm": 0.2762843370437622, + "learning_rate": 9.921323869859805e-05, + "loss": 0.007115472108125687, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00714, + "step": 219, + "tokens/total": 3306320, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 70774 + }, + { + "epoch": 3.4375, + "grad_norm": 0.24070966243743896, + "learning_rate": 9.919965590529272e-05, + "loss": 0.0015583797357976437, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00156, + "step": 220, + "tokens/total": 3321472, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 71098 + }, + { + "epoch": 3.453125, + "grad_norm": 0.10395435988903046, + "learning_rate": 9.918595791718938e-05, + "loss": 0.0007914248271845281, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00079, + "step": 221, + "tokens/total": 3336520, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 71420 + }, + { + "epoch": 3.46875, + "grad_norm": 0.5730319023132324, + "learning_rate": 9.917214476998825e-05, + "loss": 0.005572602618485689, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00559, + "step": 222, + "tokens/total": 3351528, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 71738 + }, + { + "epoch": 3.484375, + "grad_norm": 0.1355881541967392, + "learning_rate": 9.915821649968965e-05, + "loss": 0.0004524950636550784, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00045, + "step": 223, + "tokens/total": 3366648, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 72066 + }, + { + "epoch": 3.5, + "grad_norm": 0.07415846735239029, + "learning_rate": 9.914417314259398e-05, + "loss": 0.0004993464681319892, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0005, + "step": 224, + "tokens/total": 3381772, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 72393 + }, + { + "epoch": 3.515625, + "grad_norm": 0.22872294485569, + "learning_rate": 9.913001473530153e-05, + "loss": 0.0009260782971978188, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00093, + "step": 225, + "tokens/total": 3396936, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 72718 + }, + { + "epoch": 3.53125, + "grad_norm": 0.006719402503222227, + "learning_rate": 9.911574131471251e-05, + "loss": 0.0001365313510177657, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 226, + "tokens/total": 3411960, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 73041 + }, + { + "epoch": 3.546875, + "grad_norm": 0.020894810557365417, + "learning_rate": 9.910135291802682e-05, + "loss": 0.00035784178180620074, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00036, + "step": 227, + "tokens/total": 3427132, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 73368 + }, + { + "epoch": 3.5625, + "grad_norm": 0.06920765340328217, + "learning_rate": 9.908684958274405e-05, + "loss": 0.000399862794438377, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0004, + "step": 228, + "tokens/total": 3442284, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 73688 + }, + { + "epoch": 3.578125, + "grad_norm": 0.008416767232120037, + "learning_rate": 9.907223134666332e-05, + "loss": 4.7169265599222854e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00005, + "step": 229, + "tokens/total": 3457348, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 74007 + }, + { + "epoch": 3.59375, + "grad_norm": 0.013850455172359943, + "learning_rate": 9.905749824788326e-05, + "loss": 0.00019475985027384013, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00019, + "step": 230, + "tokens/total": 3472312, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 74325 + }, + { + "epoch": 3.609375, + "grad_norm": 0.0023701407480984926, + "learning_rate": 9.904265032480177e-05, + "loss": 3.626237958087586e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 231, + "tokens/total": 3487484, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 74651 + }, + { + "epoch": 3.625, + "grad_norm": 0.0015644921222701669, + "learning_rate": 9.902768761611609e-05, + "loss": 3.6560850276146084e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 232, + "tokens/total": 3502628, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 74977 + }, + { + "epoch": 3.640625, + "grad_norm": 0.54908686876297, + "learning_rate": 9.901261016082262e-05, + "loss": 0.02532411366701126, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.02565, + "step": 233, + "tokens/total": 3517796, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 75301 + }, + { + "epoch": 3.65625, + "grad_norm": 0.00214971462264657, + "learning_rate": 9.899741799821673e-05, + "loss": 3.0114068067632616e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 234, + "tokens/total": 3532868, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 75624 + }, + { + "epoch": 3.671875, + "grad_norm": 0.0071777780540287495, + "learning_rate": 9.898211116789284e-05, + "loss": 5.1465347496559843e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00005, + "step": 235, + "tokens/total": 3547960, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 75948 + }, + { + "epoch": 3.6875, + "grad_norm": 1.685914158821106, + "learning_rate": 9.896668970974419e-05, + "loss": 0.014849710278213024, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.01496, + "step": 236, + "tokens/total": 3563144, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 76270 + }, + { + "epoch": 3.703125, + "grad_norm": 1.67842698097229, + "learning_rate": 9.895115366396273e-05, + "loss": 0.0018469173228368163, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00185, + "step": 237, + "tokens/total": 3578164, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 76593 + }, + { + "epoch": 3.71875, + "grad_norm": 0.005261159967631102, + "learning_rate": 9.893550307103913e-05, + "loss": 8.17667314549908e-05, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 238, + "tokens/total": 3593340, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 76914 + }, + { + "epoch": 3.734375, + "grad_norm": 0.023177370429039, + "learning_rate": 9.891973797176252e-05, + "loss": 0.00030886579770594835, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00031, + "step": 239, + "tokens/total": 3608372, + "tokens/train_per_sec_per_gpu": 9.14, + "tokens/trainable": 77238 + }, + { + "epoch": 3.75, + "grad_norm": 0.3134678304195404, + "learning_rate": 9.890385840722048e-05, + "loss": 0.0008985826279968023, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0009, + "step": 240, + "tokens/total": 3623664, + "tokens/train_per_sec_per_gpu": 9.18, + "tokens/trainable": 77567 + }, + { + "epoch": 3.765625, + "grad_norm": 0.7450703382492065, + "learning_rate": 9.888786441879897e-05, + "loss": 0.017146039754152298, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01729, + "step": 241, + "tokens/total": 3638684, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 77893 + }, + { + "epoch": 3.78125, + "grad_norm": 0.008559435606002808, + "learning_rate": 9.887175604818206e-05, + "loss": 0.0001242949510924518, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.8, + "memory/max_allocated (GiB)": 32.8, + "ppl": 1.00012, + "step": 242, + "tokens/total": 3653632, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 78213 + }, + { + "epoch": 3.796875, + "grad_norm": 1.7312885522842407, + "learning_rate": 9.885553333735204e-05, + "loss": 0.008018076419830322, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00805, + "step": 243, + "tokens/total": 3668724, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 78536 + }, + { + "epoch": 3.8125, + "grad_norm": 1.6031489372253418, + "learning_rate": 9.883919632858914e-05, + "loss": 0.0027930131182074547, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0028, + "step": 244, + "tokens/total": 3683904, + "tokens/train_per_sec_per_gpu": 9.48, + "tokens/trainable": 78866 + }, + { + "epoch": 3.828125, + "grad_norm": 0.7863869071006775, + "learning_rate": 9.882274506447146e-05, + "loss": 0.02333652786910534, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02361, + "step": 245, + "tokens/total": 3699056, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 79196 + }, + { + "epoch": 3.84375, + "grad_norm": 0.6194532513618469, + "learning_rate": 9.88061795878749e-05, + "loss": 0.003525766544044018, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00353, + "step": 246, + "tokens/total": 3714200, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 79520 + }, + { + "epoch": 3.859375, + "grad_norm": 1.360031008720398, + "learning_rate": 9.878949994197307e-05, + "loss": 0.017261315137147903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.01741, + "step": 247, + "tokens/total": 3729364, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 79841 + }, + { + "epoch": 3.875, + "grad_norm": 0.03367060795426369, + "learning_rate": 9.877270617023703e-05, + "loss": 0.0006716687930747867, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00067, + "step": 248, + "tokens/total": 3744460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 80163 + }, + { + "epoch": 3.890625, + "grad_norm": 0.1465068757534027, + "learning_rate": 9.875579831643538e-05, + "loss": 0.0024646068923175335, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00247, + "step": 249, + "tokens/total": 3759692, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 80483 + }, + { + "epoch": 3.90625, + "grad_norm": 0.6451879739761353, + "learning_rate": 9.873877642463397e-05, + "loss": 0.016013173386454582, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01614, + "step": 250, + "tokens/total": 3774712, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 80807 + }, + { + "epoch": 3.921875, + "grad_norm": 0.5427649617195129, + "learning_rate": 9.872164053919591e-05, + "loss": 0.010463962331414223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01052, + "step": 251, + "tokens/total": 3789760, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 81131 + }, + { + "epoch": 3.9375, + "grad_norm": 0.14428693056106567, + "learning_rate": 9.870439070478138e-05, + "loss": 0.0027021225541830063, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00271, + "step": 252, + "tokens/total": 3804992, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 81459 + }, + { + "epoch": 3.953125, + "grad_norm": 0.41795286536216736, + "learning_rate": 9.868702696634756e-05, + "loss": 0.014849536120891571, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.01496, + "step": 253, + "tokens/total": 3820080, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 81785 + }, + { + "epoch": 3.96875, + "grad_norm": 0.33399561047554016, + "learning_rate": 9.866954936914845e-05, + "loss": 0.0074952589347958565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00752, + "step": 254, + "tokens/total": 3835116, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 82111 + }, + { + "epoch": 3.984375, + "grad_norm": 0.2544633150100708, + "learning_rate": 9.865195795873484e-05, + "loss": 0.005140242632478476, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00515, + "step": 255, + "tokens/total": 3850072, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 82437 + }, + { + "epoch": 4.0, + "grad_norm": 0.6971493363380432, + "learning_rate": 9.863425278095409e-05, + "loss": 0.02200455032289028, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.02225, + "step": 256, + "tokens/total": 3865312, + "tokens/train_per_sec_per_gpu": 9.43, + "tokens/trainable": 82760 + }, + { + "epoch": 4.015625, + "grad_norm": 0.15962868928909302, + "learning_rate": 9.861643388195015e-05, + "loss": 0.002751844236627221, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00276, + "step": 257, + "tokens/total": 3880436, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83083 + }, + { + "epoch": 4.03125, + "grad_norm": 0.3089817464351654, + "learning_rate": 9.859850130816326e-05, + "loss": 0.0019627867732197046, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00196, + "step": 258, + "tokens/total": 3895532, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 83405 + }, + { + "epoch": 4.046875, + "grad_norm": 0.03729058429598808, + "learning_rate": 9.858045510632996e-05, + "loss": 0.0004204344586469233, + "memory/device_reserved (GiB)": 34.1, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00042, + "step": 259, + "tokens/total": 3910816, + "tokens/train_per_sec_per_gpu": 9.19, + "tokens/trainable": 83726 + }, + { + "epoch": 4.0625, + "grad_norm": 0.2555536925792694, + "learning_rate": 9.856229532348296e-05, + "loss": 0.002585819223895669, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00259, + "step": 260, + "tokens/total": 3925864, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 84049 + }, + { + "epoch": 4.078125, + "grad_norm": 0.017389044165611267, + "learning_rate": 9.854402200695096e-05, + "loss": 0.00010104697139468044, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0001, + "step": 261, + "tokens/total": 3940976, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 84376 + }, + { + "epoch": 4.09375, + "grad_norm": 0.002870214870199561, + "learning_rate": 9.852563520435858e-05, + "loss": 5.9336227423045784e-05, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00006, + "step": 262, + "tokens/total": 3956060, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 84700 + }, + { + "epoch": 4.109375, + "grad_norm": 0.2427501380443573, + "learning_rate": 9.850713496362614e-05, + "loss": 0.004451706539839506, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.00446, + "step": 263, + "tokens/total": 3971188, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 85021 + }, + { + "epoch": 4.125, + "grad_norm": 0.005748335737735033, + "learning_rate": 9.848852133296972e-05, + "loss": 8.568367775296792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 264, + "tokens/total": 3986252, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 85344 + }, + { + "epoch": 4.140625, + "grad_norm": 0.0012240263167768717, + "learning_rate": 9.846979436090085e-05, + "loss": 4.23779092670884e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00004, + "step": 265, + "tokens/total": 4001268, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 85666 + }, + { + "epoch": 4.15625, + "grad_norm": 0.01661101169884205, + "learning_rate": 9.845095409622642e-05, + "loss": 0.0001144747220678255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00011, + "step": 266, + "tokens/total": 4016372, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 85992 + }, + { + "epoch": 4.171875, + "grad_norm": 0.020272312685847282, + "learning_rate": 9.843200058804868e-05, + "loss": 0.00017771853890735656, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00018, + "step": 267, + "tokens/total": 4031400, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 86320 + }, + { + "epoch": 4.1875, + "grad_norm": 0.033239174634218216, + "learning_rate": 9.841293388576495e-05, + "loss": 0.00044481243821792305, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00044, + "step": 268, + "tokens/total": 4046488, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 86644 + }, + { + "epoch": 4.203125, + "grad_norm": 0.5434594750404358, + "learning_rate": 9.839375403906759e-05, + "loss": 0.009829508140683174, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00988, + "step": 269, + "tokens/total": 4061536, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 86972 + }, + { + "epoch": 4.21875, + "grad_norm": 0.3308855891227722, + "learning_rate": 9.837446109794382e-05, + "loss": 0.004846261814236641, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00486, + "step": 270, + "tokens/total": 4076724, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 87296 + }, + { + "epoch": 4.234375, + "grad_norm": 0.003991277422755957, + "learning_rate": 9.835505511267567e-05, + "loss": 8.371799776796252e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00008, + "step": 271, + "tokens/total": 4091820, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 87621 + }, + { + "epoch": 4.25, + "grad_norm": 0.10116580128669739, + "learning_rate": 9.83355361338397e-05, + "loss": 0.0005483412533067167, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00055, + "step": 272, + "tokens/total": 4106976, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 87939 + }, + { + "epoch": 4.265625, + "grad_norm": 0.003369405400007963, + "learning_rate": 9.831590421230704e-05, + "loss": 7.847918459447101e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00008, + "step": 273, + "tokens/total": 4122100, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 88266 + }, + { + "epoch": 4.28125, + "grad_norm": 0.011247020214796066, + "learning_rate": 9.829615939924308e-05, + "loss": 0.000140060976264067, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00014, + "step": 274, + "tokens/total": 4137256, + "tokens/train_per_sec_per_gpu": 9.72, + "tokens/trainable": 88588 + }, + { + "epoch": 4.296875, + "grad_norm": 0.009074223227798939, + "learning_rate": 9.827630174610754e-05, + "loss": 9.554257121635601e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0001, + "step": 275, + "tokens/total": 4152376, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 88911 + }, + { + "epoch": 4.3125, + "grad_norm": 0.000541043933480978, + "learning_rate": 9.825633130465418e-05, + "loss": 1.697065818007104e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 276, + "tokens/total": 4167356, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 89237 + }, + { + "epoch": 4.328125, + "grad_norm": 0.014442048035562038, + "learning_rate": 9.823624812693069e-05, + "loss": 9.099359886022285e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00009, + "step": 277, + "tokens/total": 4182476, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 89560 + }, + { + "epoch": 4.34375, + "grad_norm": 0.013665883801877499, + "learning_rate": 9.821605226527862e-05, + "loss": 0.00011380278010619804, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00011, + "step": 278, + "tokens/total": 4197608, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 89881 + }, + { + "epoch": 4.359375, + "grad_norm": 0.0006537620793096721, + "learning_rate": 9.819574377233317e-05, + "loss": 1.6407084331149235e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00002, + "step": 279, + "tokens/total": 4212744, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 90201 + }, + { + "epoch": 4.375, + "grad_norm": 0.01291027944535017, + "learning_rate": 9.817532270102309e-05, + "loss": 6.133162969490513e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00006, + "step": 280, + "tokens/total": 4227888, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 90524 + }, + { + "epoch": 4.390625, + "grad_norm": 0.5513811111450195, + "learning_rate": 9.815478910457054e-05, + "loss": 0.007841670885682106, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00787, + "step": 281, + "tokens/total": 4242916, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 90845 + }, + { + "epoch": 4.40625, + "grad_norm": 0.0005016659270040691, + "learning_rate": 9.813414303649097e-05, + "loss": 1.6098865671665408e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 282, + "tokens/total": 4258040, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 91170 + }, + { + "epoch": 4.421875, + "grad_norm": 0.1864936798810959, + "learning_rate": 9.811338455059291e-05, + "loss": 0.0015166733646765351, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00152, + "step": 283, + "tokens/total": 4273108, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 91490 + }, + { + "epoch": 4.4375, + "grad_norm": 0.0026293513365089893, + "learning_rate": 9.809251370097791e-05, + "loss": 5.2753948693862185e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00005, + "step": 284, + "tokens/total": 4288276, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 91814 + }, + { + "epoch": 4.453125, + "grad_norm": 0.41635486483573914, + "learning_rate": 9.807153054204036e-05, + "loss": 0.02190607599914074, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.02215, + "step": 285, + "tokens/total": 4303308, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 92134 + }, + { + "epoch": 4.46875, + "grad_norm": 1.0917211771011353, + "learning_rate": 9.805043512846735e-05, + "loss": 0.019963297992944717, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02016, + "step": 286, + "tokens/total": 4318444, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 92457 + }, + { + "epoch": 4.484375, + "grad_norm": 0.06857825815677643, + "learning_rate": 9.802922751523855e-05, + "loss": 0.00030228254036046565, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0003, + "step": 287, + "tokens/total": 4333612, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 92781 + }, + { + "epoch": 4.5, + "grad_norm": 0.031100548803806305, + "learning_rate": 9.800790775762605e-05, + "loss": 0.0004046057292725891, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0004, + "step": 288, + "tokens/total": 4348684, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 93103 + }, + { + "epoch": 4.515625, + "grad_norm": 0.47315219044685364, + "learning_rate": 9.798647591119415e-05, + "loss": 0.0014745643129572272, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00148, + "step": 289, + "tokens/total": 4363864, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93425 + }, + { + "epoch": 4.53125, + "grad_norm": 0.0773354172706604, + "learning_rate": 9.796493203179937e-05, + "loss": 0.0008912066696211696, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00089, + "step": 290, + "tokens/total": 4379016, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 93751 + }, + { + "epoch": 4.546875, + "grad_norm": 0.1289902925491333, + "learning_rate": 9.794327617559016e-05, + "loss": 0.0018994644051417708, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0019, + "step": 291, + "tokens/total": 4394032, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 94073 + }, + { + "epoch": 4.5625, + "grad_norm": 0.054183945059776306, + "learning_rate": 9.792150839900684e-05, + "loss": 0.0005117067485116422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00051, + "step": 292, + "tokens/total": 4409044, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 94394 + }, + { + "epoch": 4.578125, + "grad_norm": 0.12445089966058731, + "learning_rate": 9.789962875878139e-05, + "loss": 0.0022513207513839006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00225, + "step": 293, + "tokens/total": 4424184, + "tokens/train_per_sec_per_gpu": 9.98, + "tokens/trainable": 94721 + }, + { + "epoch": 4.59375, + "grad_norm": 0.20701012015342712, + "learning_rate": 9.787763731193733e-05, + "loss": 0.003331368323415518, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00334, + "step": 294, + "tokens/total": 4439260, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 95042 + }, + { + "epoch": 4.609375, + "grad_norm": 0.6194630265235901, + "learning_rate": 9.785553411578963e-05, + "loss": 0.03162220120429993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.03213, + "step": 295, + "tokens/total": 4454416, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 95363 + }, + { + "epoch": 4.625, + "grad_norm": 0.2134065181016922, + "learning_rate": 9.783331922794444e-05, + "loss": 0.0037704987917095423, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00378, + "step": 296, + "tokens/total": 4469496, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 95682 + }, + { + "epoch": 4.640625, + "grad_norm": 0.040715303272008896, + "learning_rate": 9.781099270629905e-05, + "loss": 0.0002603174070827663, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00026, + "step": 297, + "tokens/total": 4484496, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 96007 + }, + { + "epoch": 4.65625, + "grad_norm": 0.7002236247062683, + "learning_rate": 9.778855460904167e-05, + "loss": 0.006636213511228561, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00666, + "step": 298, + "tokens/total": 4499572, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 96333 + }, + { + "epoch": 4.671875, + "grad_norm": 0.007663452997803688, + "learning_rate": 9.77660049946513e-05, + "loss": 0.00024576610303483903, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00025, + "step": 299, + "tokens/total": 4514704, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 96657 + }, + { + "epoch": 4.6875, + "grad_norm": 0.023272940889000893, + "learning_rate": 9.77433439218976e-05, + "loss": 0.0005693582352250814, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00057, + "step": 300, + "tokens/total": 4529808, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 96980 + }, + { + "epoch": 4.703125, + "grad_norm": 0.026790525764226913, + "learning_rate": 9.77205714498407e-05, + "loss": 0.0007213435601443052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00072, + "step": 301, + "tokens/total": 4544924, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 97298 + }, + { + "epoch": 4.71875, + "grad_norm": 0.004755170550197363, + "learning_rate": 9.769768763783107e-05, + "loss": 0.00017550830671098083, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00018, + "step": 302, + "tokens/total": 4559952, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 97625 + }, + { + "epoch": 4.734375, + "grad_norm": 0.016630757600069046, + "learning_rate": 9.767469254550936e-05, + "loss": 0.0002396691415924579, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00024, + "step": 303, + "tokens/total": 4575048, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 97951 + }, + { + "epoch": 4.75, + "grad_norm": 0.09570515155792236, + "learning_rate": 9.765158623280625e-05, + "loss": 0.0009529241942800581, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00095, + "step": 304, + "tokens/total": 4590200, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 98274 + }, + { + "epoch": 4.765625, + "grad_norm": 0.5903910994529724, + "learning_rate": 9.762836875994225e-05, + "loss": 0.005048507358878851, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00506, + "step": 305, + "tokens/total": 4605312, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 98599 + }, + { + "epoch": 4.78125, + "grad_norm": 0.008239318616688251, + "learning_rate": 9.760504018742764e-05, + "loss": 0.00023036239144857973, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00023, + "step": 306, + "tokens/total": 4620364, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 98923 + }, + { + "epoch": 4.796875, + "grad_norm": 0.060113124549388885, + "learning_rate": 9.75816005760622e-05, + "loss": 0.0003171982243657112, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00032, + "step": 307, + "tokens/total": 4635520, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 99245 + }, + { + "epoch": 4.8125, + "grad_norm": 0.008316988125443459, + "learning_rate": 9.755804998693513e-05, + "loss": 0.00014665098569821566, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00015, + "step": 308, + "tokens/total": 4650576, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 99567 + }, + { + "epoch": 4.828125, + "grad_norm": 0.21268479526042938, + "learning_rate": 9.753438848142485e-05, + "loss": 0.0025118833873420954, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00252, + "step": 309, + "tokens/total": 4665688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 99884 + }, + { + "epoch": 4.84375, + "grad_norm": 0.005383700132369995, + "learning_rate": 9.751061612119887e-05, + "loss": 0.00013681002019438893, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00014, + "step": 310, + "tokens/total": 4680792, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 100210 + }, + { + "epoch": 4.859375, + "grad_norm": 0.4226222336292267, + "learning_rate": 9.748673296821358e-05, + "loss": 0.00028868031222373247, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00029, + "step": 311, + "tokens/total": 4695976, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 100531 + }, + { + "epoch": 4.875, + "grad_norm": 0.007631849963217974, + "learning_rate": 9.746273908471418e-05, + "loss": 0.000154903216753155, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00015, + "step": 312, + "tokens/total": 4711032, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 100855 + }, + { + "epoch": 4.890625, + "grad_norm": 0.09700846672058105, + "learning_rate": 9.743863453323441e-05, + "loss": 0.0013645484577864408, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00137, + "step": 313, + "tokens/total": 4726120, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 101178 + }, + { + "epoch": 4.90625, + "grad_norm": 0.018550626933574677, + "learning_rate": 9.741441937659645e-05, + "loss": 0.0002988259366247803, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0003, + "step": 314, + "tokens/total": 4741180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 101505 + }, + { + "epoch": 4.921875, + "grad_norm": 0.005299921613186598, + "learning_rate": 9.73900936779108e-05, + "loss": 8.563048322685063e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 315, + "tokens/total": 4756212, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 101826 + }, + { + "epoch": 4.9375, + "grad_norm": 0.015369605273008347, + "learning_rate": 9.736565750057593e-05, + "loss": 0.00014232251851353794, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 316, + "tokens/total": 4771304, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 102145 + }, + { + "epoch": 4.953125, + "grad_norm": 0.006885336712002754, + "learning_rate": 9.734111090827836e-05, + "loss": 0.00014437930076383054, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00014, + "step": 317, + "tokens/total": 4786368, + "tokens/train_per_sec_per_gpu": 9.81, + "tokens/trainable": 102474 + }, + { + "epoch": 4.96875, + "grad_norm": 0.3322601020336151, + "learning_rate": 9.731645396499232e-05, + "loss": 0.0058242324739694595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00584, + "step": 318, + "tokens/total": 4801584, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 102799 + }, + { + "epoch": 4.984375, + "grad_norm": 0.008417945355176926, + "learning_rate": 9.729168673497969e-05, + "loss": 9.954217239283025e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0001, + "step": 319, + "tokens/total": 4816636, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 103127 + }, + { + "epoch": 5.0, + "grad_norm": 0.3274463713169098, + "learning_rate": 9.726680928278972e-05, + "loss": 0.0031569483689963818, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00316, + "step": 320, + "tokens/total": 4831664, + "tokens/train_per_sec_per_gpu": 9.84, + "tokens/trainable": 103450 + }, + { + "epoch": 5.015625, + "grad_norm": 0.011693589389324188, + "learning_rate": 9.724182167325899e-05, + "loss": 7.778844883432612e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00008, + "step": 321, + "tokens/total": 4846820, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 103775 + }, + { + "epoch": 5.03125, + "grad_norm": 1.3683384656906128, + "learning_rate": 9.721672397151109e-05, + "loss": 0.010511158034205437, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01057, + "step": 322, + "tokens/total": 4861864, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 104098 + }, + { + "epoch": 5.046875, + "grad_norm": 0.005915132351219654, + "learning_rate": 9.719151624295665e-05, + "loss": 6.681385275442153e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00007, + "step": 323, + "tokens/total": 4876928, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104423 + }, + { + "epoch": 5.0625, + "grad_norm": 0.00782559160143137, + "learning_rate": 9.716619855329296e-05, + "loss": 8.550586790079251e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00009, + "step": 324, + "tokens/total": 4892060, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 104745 + }, + { + "epoch": 5.078125, + "grad_norm": 0.0015690920408815145, + "learning_rate": 9.714077096850394e-05, + "loss": 2.1982301404932514e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 325, + "tokens/total": 4907184, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 105073 + }, + { + "epoch": 5.09375, + "grad_norm": 0.0027190272230654955, + "learning_rate": 9.711523355485992e-05, + "loss": 2.6376750611234456e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 326, + "tokens/total": 4922316, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 105398 + }, + { + "epoch": 5.109375, + "grad_norm": 0.21495184302330017, + "learning_rate": 9.708958637891746e-05, + "loss": 0.0006077390280552208, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00061, + "step": 327, + "tokens/total": 4937428, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 105717 + }, + { + "epoch": 5.125, + "grad_norm": 0.0004555387713480741, + "learning_rate": 9.706382950751918e-05, + "loss": 1.7097307136282325e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00002, + "step": 328, + "tokens/total": 4952516, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 106042 + }, + { + "epoch": 5.140625, + "grad_norm": 0.02135647088289261, + "learning_rate": 9.703796300779361e-05, + "loss": 0.00014291358820628375, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00014, + "step": 329, + "tokens/total": 4967424, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 106368 + }, + { + "epoch": 5.15625, + "grad_norm": 0.011736172251403332, + "learning_rate": 9.7011986947155e-05, + "loss": 3.78161967091728e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00004, + "step": 330, + "tokens/total": 4982504, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 106694 + }, + { + "epoch": 5.171875, + "grad_norm": 0.874617338180542, + "learning_rate": 9.698590139330309e-05, + "loss": 0.006245121825486422, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00626, + "step": 331, + "tokens/total": 4997640, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 107024 + }, + { + "epoch": 5.1875, + "grad_norm": 0.027552802115678787, + "learning_rate": 9.695970641422309e-05, + "loss": 0.00019723488367162645, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0002, + "step": 332, + "tokens/total": 5012832, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 107343 + }, + { + "epoch": 5.203125, + "grad_norm": 0.00035697652492672205, + "learning_rate": 9.693340207818528e-05, + "loss": 1.3179367670090869e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 333, + "tokens/total": 5027964, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 107673 + }, + { + "epoch": 5.21875, + "grad_norm": 0.08428651839494705, + "learning_rate": 9.690698845374505e-05, + "loss": 0.0008772820001468062, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00088, + "step": 334, + "tokens/total": 5042980, + "tokens/train_per_sec_per_gpu": 10.09, + "tokens/trainable": 108000 + }, + { + "epoch": 5.234375, + "grad_norm": 0.00604958226904273, + "learning_rate": 9.688046560974256e-05, + "loss": 8.847641583997756e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00009, + "step": 335, + "tokens/total": 5058152, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 108327 + }, + { + "epoch": 5.25, + "grad_norm": 0.00039003140409477055, + "learning_rate": 9.685383361530262e-05, + "loss": 1.5304913176805712e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 336, + "tokens/total": 5073280, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 108651 + }, + { + "epoch": 5.265625, + "grad_norm": 0.00023623408924322575, + "learning_rate": 9.682709253983455e-05, + "loss": 8.844612239045091e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00001, + "step": 337, + "tokens/total": 5088336, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 108972 + }, + { + "epoch": 5.28125, + "grad_norm": 0.00048069472541101277, + "learning_rate": 9.680024245303193e-05, + "loss": 1.667063588683959e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 338, + "tokens/total": 5103484, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 109297 + }, + { + "epoch": 5.296875, + "grad_norm": 0.16455329954624176, + "learning_rate": 9.677328342487248e-05, + "loss": 0.0005412459140643477, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00054, + "step": 339, + "tokens/total": 5118608, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 109618 + }, + { + "epoch": 5.3125, + "grad_norm": 0.012845818884670734, + "learning_rate": 9.674621552561782e-05, + "loss": 0.00013661464618053287, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00014, + "step": 340, + "tokens/total": 5133580, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 109940 + }, + { + "epoch": 5.328125, + "grad_norm": 0.7110337615013123, + "learning_rate": 9.671903882581331e-05, + "loss": 0.005711458623409271, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00573, + "step": 341, + "tokens/total": 5148700, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 110268 + }, + { + "epoch": 5.34375, + "grad_norm": 0.007893380708992481, + "learning_rate": 9.669175339628791e-05, + "loss": 9.75146540440619e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0001, + "step": 342, + "tokens/total": 5163756, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 110592 + }, + { + "epoch": 5.359375, + "grad_norm": 0.10780492424964905, + "learning_rate": 9.66643593081539e-05, + "loss": 0.0005473099881783128, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00055, + "step": 343, + "tokens/total": 5178780, + "tokens/train_per_sec_per_gpu": 9.63, + "tokens/trainable": 110912 + }, + { + "epoch": 5.375, + "grad_norm": 0.0010501379147171974, + "learning_rate": 9.663685663280679e-05, + "loss": 2.7556767236092128e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 344, + "tokens/total": 5193856, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 111238 + }, + { + "epoch": 5.390625, + "grad_norm": 0.288404256105423, + "learning_rate": 9.66092454419251e-05, + "loss": 0.0018175432924181223, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00182, + "step": 345, + "tokens/total": 5208940, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 111557 + }, + { + "epoch": 5.40625, + "grad_norm": 1.9734091758728027, + "learning_rate": 9.65815258074701e-05, + "loss": 0.03248640149831772, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03302, + "step": 346, + "tokens/total": 5224084, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 111879 + }, + { + "epoch": 5.421875, + "grad_norm": 0.2764054536819458, + "learning_rate": 9.655369780168579e-05, + "loss": 0.0009657267364673316, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00097, + "step": 347, + "tokens/total": 5239172, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 112201 + }, + { + "epoch": 5.4375, + "grad_norm": 0.0005600836011581123, + "learning_rate": 9.652576149709852e-05, + "loss": 1.6435071302112192e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 348, + "tokens/total": 5254272, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 112522 + }, + { + "epoch": 5.453125, + "grad_norm": 0.0008129188208840787, + "learning_rate": 9.649771696651697e-05, + "loss": 2.3179851268650964e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00002, + "step": 349, + "tokens/total": 5269472, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 112844 + }, + { + "epoch": 5.46875, + "grad_norm": 0.0014548521721735597, + "learning_rate": 9.646956428303183e-05, + "loss": 1.6358961147489026e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 350, + "tokens/total": 5284592, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 113170 + }, + { + "epoch": 5.484375, + "grad_norm": 0.5601457953453064, + "learning_rate": 9.644130352001567e-05, + "loss": 0.003099913941696286, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.0031, + "step": 351, + "tokens/total": 5299784, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 113496 + }, + { + "epoch": 5.5, + "grad_norm": 0.00297771324403584, + "learning_rate": 9.641293475112275e-05, + "loss": 4.133052425459027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00004, + "step": 352, + "tokens/total": 5314884, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 113820 + }, + { + "epoch": 5.515625, + "grad_norm": 0.0016736306715756655, + "learning_rate": 9.638445805028884e-05, + "loss": 4.893036748399027e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00005, + "step": 353, + "tokens/total": 5330048, + "tokens/train_per_sec_per_gpu": 9.73, + "tokens/trainable": 114143 + }, + { + "epoch": 5.53125, + "grad_norm": 0.001026933896355331, + "learning_rate": 9.635587349173095e-05, + "loss": 2.9338965759961866e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00003, + "step": 354, + "tokens/total": 5345196, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 114470 + }, + { + "epoch": 5.546875, + "grad_norm": 0.008905365131795406, + "learning_rate": 9.632718114994726e-05, + "loss": 9.410006168764085e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 355, + "tokens/total": 5360188, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 114793 + }, + { + "epoch": 5.5625, + "grad_norm": 0.44935929775238037, + "learning_rate": 9.62983810997168e-05, + "loss": 0.0031317835673689842, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00314, + "step": 356, + "tokens/total": 5375152, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 115118 + }, + { + "epoch": 5.578125, + "grad_norm": 0.004555090330541134, + "learning_rate": 9.626947341609934e-05, + "loss": 0.00011855485354317352, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00012, + "step": 357, + "tokens/total": 5390232, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 115437 + }, + { + "epoch": 5.59375, + "grad_norm": 0.001394708058796823, + "learning_rate": 9.624045817443519e-05, + "loss": 4.678318146034144e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00005, + "step": 358, + "tokens/total": 5405308, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 115755 + }, + { + "epoch": 5.609375, + "grad_norm": 0.0037189542781561613, + "learning_rate": 9.621133545034491e-05, + "loss": 0.00011213941616006196, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00011, + "step": 359, + "tokens/total": 5420468, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 116074 + }, + { + "epoch": 5.625, + "grad_norm": 0.1473202258348465, + "learning_rate": 9.61821053197293e-05, + "loss": 0.0005101395072415471, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00051, + "step": 360, + "tokens/total": 5435636, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 116397 + }, + { + "epoch": 5.640625, + "grad_norm": 0.012220866046845913, + "learning_rate": 9.615276785876898e-05, + "loss": 0.0002749463019426912, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00027, + "step": 361, + "tokens/total": 5450808, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 116719 + }, + { + "epoch": 5.65625, + "grad_norm": 0.007556390482932329, + "learning_rate": 9.612332314392435e-05, + "loss": 0.00013154096086509526, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00013, + "step": 362, + "tokens/total": 5465840, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 117047 + }, + { + "epoch": 5.671875, + "grad_norm": 0.002842485439032316, + "learning_rate": 9.609377125193535e-05, + "loss": 7.768635987304151e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00008, + "step": 363, + "tokens/total": 5480852, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 117370 + }, + { + "epoch": 5.6875, + "grad_norm": 0.04467956721782684, + "learning_rate": 9.606411225982118e-05, + "loss": 0.00022737993276678026, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00023, + "step": 364, + "tokens/total": 5495908, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 117690 + }, + { + "epoch": 5.703125, + "grad_norm": 1.0502334833145142, + "learning_rate": 9.603434624488023e-05, + "loss": 0.015947995707392693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.01608, + "step": 365, + "tokens/total": 5511044, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 118013 + }, + { + "epoch": 5.71875, + "grad_norm": 0.014036186039447784, + "learning_rate": 9.600447328468982e-05, + "loss": 0.00013877129822503775, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 366, + "tokens/total": 5526156, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 118337 + }, + { + "epoch": 5.734375, + "grad_norm": 0.6235236525535583, + "learning_rate": 9.597449345710597e-05, + "loss": 0.015155073255300522, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.01527, + "step": 367, + "tokens/total": 5541220, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 118656 + }, + { + "epoch": 5.75, + "grad_norm": 0.0999632477760315, + "learning_rate": 9.594440684026324e-05, + "loss": 0.0012880185386165977, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00129, + "step": 368, + "tokens/total": 5556260, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 118978 + }, + { + "epoch": 5.765625, + "grad_norm": 0.02786194533109665, + "learning_rate": 9.591421351257449e-05, + "loss": 0.0003391870704945177, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00034, + "step": 369, + "tokens/total": 5571400, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 119301 + }, + { + "epoch": 5.78125, + "grad_norm": 0.5290789008140564, + "learning_rate": 9.588391355273066e-05, + "loss": 0.007406376302242279, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00743, + "step": 370, + "tokens/total": 5586416, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 119622 + }, + { + "epoch": 5.796875, + "grad_norm": 0.01936807669699192, + "learning_rate": 9.585350703970069e-05, + "loss": 0.00025529557024128735, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00026, + "step": 371, + "tokens/total": 5601532, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 119945 + }, + { + "epoch": 5.8125, + "grad_norm": 0.005768527276813984, + "learning_rate": 9.582299405273118e-05, + "loss": 8.689028618391603e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00009, + "step": 372, + "tokens/total": 5616504, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 120265 + }, + { + "epoch": 5.828125, + "grad_norm": 0.9624814391136169, + "learning_rate": 9.579237467134617e-05, + "loss": 0.00811285525560379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00815, + "step": 373, + "tokens/total": 5631576, + "tokens/train_per_sec_per_gpu": 9.38, + "tokens/trainable": 120582 + }, + { + "epoch": 5.84375, + "grad_norm": 0.018384141847491264, + "learning_rate": 9.576164897534709e-05, + "loss": 0.0003791853378061205, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00038, + "step": 374, + "tokens/total": 5646596, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 120906 + }, + { + "epoch": 5.859375, + "grad_norm": 0.6165330410003662, + "learning_rate": 9.573081704481236e-05, + "loss": 0.013609700836241245, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0137, + "step": 375, + "tokens/total": 5661732, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 121231 + }, + { + "epoch": 5.875, + "grad_norm": 0.0840073898434639, + "learning_rate": 9.569987896009734e-05, + "loss": 0.00045918801333755255, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00046, + "step": 376, + "tokens/total": 5676732, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 121550 + }, + { + "epoch": 5.890625, + "grad_norm": 0.008723951876163483, + "learning_rate": 9.566883480183399e-05, + "loss": 0.00011756666208384559, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00012, + "step": 377, + "tokens/total": 5691812, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 121874 + }, + { + "epoch": 5.90625, + "grad_norm": 0.012961777858436108, + "learning_rate": 9.563768465093079e-05, + "loss": 0.00030332955066114664, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0003, + "step": 378, + "tokens/total": 5706864, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 122202 + }, + { + "epoch": 5.921875, + "grad_norm": 0.02426735870540142, + "learning_rate": 9.560642858857243e-05, + "loss": 0.00048618094297125936, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00049, + "step": 379, + "tokens/total": 5721984, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 122525 + }, + { + "epoch": 5.9375, + "grad_norm": 0.1119721531867981, + "learning_rate": 9.55750666962196e-05, + "loss": 0.0008597768610343337, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00086, + "step": 380, + "tokens/total": 5737156, + "tokens/train_per_sec_per_gpu": 9.77, + "tokens/trainable": 122847 + }, + { + "epoch": 5.953125, + "grad_norm": 0.03705863282084465, + "learning_rate": 9.554359905560886e-05, + "loss": 0.0003950979153160006, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0004, + "step": 381, + "tokens/total": 5752164, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 123170 + }, + { + "epoch": 5.96875, + "grad_norm": 0.6851837038993835, + "learning_rate": 9.551202574875235e-05, + "loss": 0.011227660812437534, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.01129, + "step": 382, + "tokens/total": 5767272, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 123495 + }, + { + "epoch": 5.984375, + "grad_norm": 0.06478452682495117, + "learning_rate": 9.548034685793762e-05, + "loss": 0.0012987275840714574, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.0013, + "step": 383, + "tokens/total": 5782328, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 123819 + }, + { + "epoch": 6.0, + "grad_norm": 0.886103630065918, + "learning_rate": 9.544856246572735e-05, + "loss": 0.019783541560173035, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.01998, + "step": 384, + "tokens/total": 5797392, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 124140 + }, + { + "epoch": 6.015625, + "grad_norm": 0.03494488075375557, + "learning_rate": 9.541667265495922e-05, + "loss": 0.0006417686236090958, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00064, + "step": 385, + "tokens/total": 5812552, + "tokens/train_per_sec_per_gpu": 9.03, + "tokens/trainable": 124465 + }, + { + "epoch": 6.03125, + "grad_norm": 0.1211373507976532, + "learning_rate": 9.538467750874569e-05, + "loss": 0.0016129965661093593, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00161, + "step": 386, + "tokens/total": 5827696, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 124785 + }, + { + "epoch": 6.046875, + "grad_norm": 0.9197566509246826, + "learning_rate": 9.535257711047364e-05, + "loss": 0.01892932318150997, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.01911, + "step": 387, + "tokens/total": 5842804, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 125115 + }, + { + "epoch": 6.0625, + "grad_norm": 0.3712734878063202, + "learning_rate": 9.532037154380439e-05, + "loss": 0.004730743821710348, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00474, + "step": 388, + "tokens/total": 5857812, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 125435 + }, + { + "epoch": 6.078125, + "grad_norm": 0.13118499517440796, + "learning_rate": 9.528806089267325e-05, + "loss": 0.0026065458077937365, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00261, + "step": 389, + "tokens/total": 5872820, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 125757 + }, + { + "epoch": 6.09375, + "grad_norm": 0.41263192892074585, + "learning_rate": 9.525564524128947e-05, + "loss": 0.02503504976630211, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.02535, + "step": 390, + "tokens/total": 5887900, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 126077 + }, + { + "epoch": 6.109375, + "grad_norm": 0.05777617543935776, + "learning_rate": 9.522312467413594e-05, + "loss": 0.0012999713653698564, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0013, + "step": 391, + "tokens/total": 5903096, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 126401 + }, + { + "epoch": 6.125, + "grad_norm": 0.16045935451984406, + "learning_rate": 9.519049927596896e-05, + "loss": 0.006284795701503754, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0063, + "step": 392, + "tokens/total": 5918200, + "tokens/train_per_sec_per_gpu": 10.05, + "tokens/trainable": 126731 + }, + { + "epoch": 6.140625, + "grad_norm": 0.10557752847671509, + "learning_rate": 9.515776913181807e-05, + "loss": 0.0018496124539524317, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00185, + "step": 393, + "tokens/total": 5933300, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 127056 + }, + { + "epoch": 6.15625, + "grad_norm": 0.1812046468257904, + "learning_rate": 9.512493432698582e-05, + "loss": 0.003738866187632084, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00375, + "step": 394, + "tokens/total": 5948468, + "tokens/train_per_sec_per_gpu": 9.05, + "tokens/trainable": 127377 + }, + { + "epoch": 6.171875, + "grad_norm": 0.11412262171506882, + "learning_rate": 9.509199494704748e-05, + "loss": 0.00221292139030993, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.00222, + "step": 395, + "tokens/total": 5963588, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 127698 + }, + { + "epoch": 6.1875, + "grad_norm": 0.05267500877380371, + "learning_rate": 9.505895107785091e-05, + "loss": 0.0010225172154605389, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00102, + "step": 396, + "tokens/total": 5978744, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 128021 + }, + { + "epoch": 6.203125, + "grad_norm": 0.23904849588871002, + "learning_rate": 9.502580280551627e-05, + "loss": 0.005414942279458046, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00543, + "step": 397, + "tokens/total": 5993864, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 128346 + }, + { + "epoch": 6.21875, + "grad_norm": 0.013456333428621292, + "learning_rate": 9.499255021643582e-05, + "loss": 0.00013312777446117252, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00013, + "step": 398, + "tokens/total": 6009136, + "tokens/train_per_sec_per_gpu": 9.7, + "tokens/trainable": 128673 + }, + { + "epoch": 6.234375, + "grad_norm": 0.0178237147629261, + "learning_rate": 9.495919339727372e-05, + "loss": 0.0003349714388605207, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00034, + "step": 399, + "tokens/total": 6024180, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 128997 + }, + { + "epoch": 6.25, + "grad_norm": 0.049912769347429276, + "learning_rate": 9.492573243496578e-05, + "loss": 0.0004279202257748693, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00043, + "step": 400, + "tokens/total": 6039336, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 129322 + }, + { + "epoch": 6.265625, + "grad_norm": 0.014288226142525673, + "learning_rate": 9.489216741671916e-05, + "loss": 0.00036869742325507104, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00037, + "step": 401, + "tokens/total": 6054380, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 129646 + }, + { + "epoch": 6.28125, + "grad_norm": 0.00872453860938549, + "learning_rate": 9.485849843001232e-05, + "loss": 6.138686876511201e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00006, + "step": 402, + "tokens/total": 6069472, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 129968 + }, + { + "epoch": 6.296875, + "grad_norm": 0.0013658624375239015, + "learning_rate": 9.48247255625946e-05, + "loss": 3.532581831677817e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00004, + "step": 403, + "tokens/total": 6084532, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 130291 + }, + { + "epoch": 6.3125, + "grad_norm": 0.0017636138945817947, + "learning_rate": 9.479084890248611e-05, + "loss": 2.53205780609278e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00003, + "step": 404, + "tokens/total": 6099556, + "tokens/train_per_sec_per_gpu": 9.16, + "tokens/trainable": 130612 + }, + { + "epoch": 6.328125, + "grad_norm": 0.000683065562043339, + "learning_rate": 9.475686853797749e-05, + "loss": 1.986018833122216e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00002, + "step": 405, + "tokens/total": 6114688, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 130935 + }, + { + "epoch": 6.34375, + "grad_norm": 0.016209598630666733, + "learning_rate": 9.472278455762962e-05, + "loss": 0.00014525593724101782, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00015, + "step": 406, + "tokens/total": 6129764, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 131259 + }, + { + "epoch": 6.359375, + "grad_norm": 0.020629821345210075, + "learning_rate": 9.468859705027345e-05, + "loss": 8.353606972377747e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00008, + "step": 407, + "tokens/total": 6144864, + "tokens/train_per_sec_per_gpu": 9.09, + "tokens/trainable": 131581 + }, + { + "epoch": 6.375, + "grad_norm": 0.0009277141070924699, + "learning_rate": 9.465430610500977e-05, + "loss": 1.4405310139409266e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 408, + "tokens/total": 6159936, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 131902 + }, + { + "epoch": 6.390625, + "grad_norm": 0.0002464439603500068, + "learning_rate": 9.461991181120889e-05, + "loss": 9.015106115839444e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 409, + "tokens/total": 6174992, + "tokens/train_per_sec_per_gpu": 9.79, + "tokens/trainable": 132228 + }, + { + "epoch": 6.40625, + "grad_norm": 0.000667985004838556, + "learning_rate": 9.458541425851051e-05, + "loss": 2.003746703849174e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00002, + "step": 410, + "tokens/total": 6190088, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 132552 + }, + { + "epoch": 6.421875, + "grad_norm": 0.002300550462678075, + "learning_rate": 9.455081353682347e-05, + "loss": 4.4835043809143826e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.97, + "memory/max_allocated (GiB)": 32.97, + "ppl": 1.00004, + "step": 411, + "tokens/total": 6205268, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 132876 + }, + { + "epoch": 6.4375, + "grad_norm": 0.0017511912155896425, + "learning_rate": 9.451610973632543e-05, + "loss": 6.716565621900372e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00001, + "step": 412, + "tokens/total": 6220344, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 133198 + }, + { + "epoch": 6.453125, + "grad_norm": 0.0028233909979462624, + "learning_rate": 9.448130294746276e-05, + "loss": 2.4473858502460644e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 413, + "tokens/total": 6235392, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 133523 + }, + { + "epoch": 6.46875, + "grad_norm": 0.00019360017904546112, + "learning_rate": 9.444639326095022e-05, + "loss": 5.723140020563733e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 414, + "tokens/total": 6250444, + "tokens/train_per_sec_per_gpu": 8.88, + "tokens/trainable": 133840 + }, + { + "epoch": 6.484375, + "grad_norm": 0.00023336702724918723, + "learning_rate": 9.441138076777071e-05, + "loss": 4.722754511021776e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.0, + "step": 415, + "tokens/total": 6265460, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 134163 + }, + { + "epoch": 6.5, + "grad_norm": 0.0003432357043493539, + "learning_rate": 9.437626555917512e-05, + "loss": 9.122491974267177e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 416, + "tokens/total": 6280524, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 134487 + }, + { + "epoch": 6.515625, + "grad_norm": 0.23666708171367645, + "learning_rate": 9.434104772668203e-05, + "loss": 0.0011873405892401934, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00119, + "step": 417, + "tokens/total": 6295592, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 134816 + }, + { + "epoch": 6.53125, + "grad_norm": 0.0001242521102540195, + "learning_rate": 9.430572736207746e-05, + "loss": 2.8345539249130525e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 418, + "tokens/total": 6310588, + "tokens/train_per_sec_per_gpu": 9.13, + "tokens/trainable": 135139 + }, + { + "epoch": 6.546875, + "grad_norm": 0.00043305609142407775, + "learning_rate": 9.427030455741466e-05, + "loss": 1.0893121725530364e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00001, + "step": 419, + "tokens/total": 6325528, + "tokens/train_per_sec_per_gpu": 9.4, + "tokens/trainable": 135463 + }, + { + "epoch": 6.5625, + "grad_norm": 0.005291244946420193, + "learning_rate": 9.423477940501391e-05, + "loss": 3.527114313328639e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00004, + "step": 420, + "tokens/total": 6340624, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 135785 + }, + { + "epoch": 6.578125, + "grad_norm": 0.0002391930902376771, + "learning_rate": 9.419915199746214e-05, + "loss": 4.351905317889759e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 421, + "tokens/total": 6355908, + "tokens/train_per_sec_per_gpu": 9.2, + "tokens/trainable": 136115 + }, + { + "epoch": 6.59375, + "grad_norm": 0.0001795155694708228, + "learning_rate": 9.416342242761287e-05, + "loss": 4.172787157585844e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 422, + "tokens/total": 6370976, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 136434 + }, + { + "epoch": 6.609375, + "grad_norm": 0.00027239706832915545, + "learning_rate": 9.412759078858583e-05, + "loss": 6.559062967426144e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 423, + "tokens/total": 6386160, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 136759 + }, + { + "epoch": 6.625, + "grad_norm": 0.004924969747662544, + "learning_rate": 9.409165717376678e-05, + "loss": 1.9311535652377643e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00002, + "step": 424, + "tokens/total": 6401172, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 137081 + }, + { + "epoch": 6.640625, + "grad_norm": 0.00028949262923561037, + "learning_rate": 9.405562167680723e-05, + "loss": 5.919015166000463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00001, + "step": 425, + "tokens/total": 6416396, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 137407 + }, + { + "epoch": 6.65625, + "grad_norm": 0.0019513516454026103, + "learning_rate": 9.401948439162429e-05, + "loss": 6.4454043240402825e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00001, + "step": 426, + "tokens/total": 6431464, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 137729 + }, + { + "epoch": 6.671875, + "grad_norm": 0.00024267581466119736, + "learning_rate": 9.398324541240028e-05, + "loss": 2.6868240183830494e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 427, + "tokens/total": 6446500, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 138055 + }, + { + "epoch": 6.6875, + "grad_norm": 0.0005200989544391632, + "learning_rate": 9.394690483358256e-05, + "loss": 5.118726221553516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 428, + "tokens/total": 6461656, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 138379 + }, + { + "epoch": 6.703125, + "grad_norm": 0.00012520125892478973, + "learning_rate": 9.391046274988333e-05, + "loss": 2.580744194347062e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 429, + "tokens/total": 6476812, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 138696 + }, + { + "epoch": 6.71875, + "grad_norm": 0.00037211537710390985, + "learning_rate": 9.387391925627933e-05, + "loss": 6.75070896249963e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 430, + "tokens/total": 6491956, + "tokens/train_per_sec_per_gpu": 10.17, + "tokens/trainable": 139025 + }, + { + "epoch": 6.734375, + "grad_norm": 0.006821396294981241, + "learning_rate": 9.383727444801156e-05, + "loss": 1.0072410987049807e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00001, + "step": 431, + "tokens/total": 6507108, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 139344 + }, + { + "epoch": 6.75, + "grad_norm": 0.005923285149037838, + "learning_rate": 9.380052842058508e-05, + "loss": 1.7001904780045152e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 432, + "tokens/total": 6522272, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 139666 + }, + { + "epoch": 6.765625, + "grad_norm": 0.0007364701014012098, + "learning_rate": 9.376368126976879e-05, + "loss": 6.04315209784545e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 433, + "tokens/total": 6537280, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 139988 + }, + { + "epoch": 6.78125, + "grad_norm": 0.0021281663794070482, + "learning_rate": 9.37267330915951e-05, + "loss": 4.2355783079983667e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 434, + "tokens/total": 6552432, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 140309 + }, + { + "epoch": 6.796875, + "grad_norm": 0.00013200279499869794, + "learning_rate": 9.368968398235973e-05, + "loss": 2.4993578335852362e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 435, + "tokens/total": 6567456, + "tokens/train_per_sec_per_gpu": 8.62, + "tokens/trainable": 140629 + }, + { + "epoch": 6.8125, + "grad_norm": 9.607620449969545e-05, + "learning_rate": 9.365253403862147e-05, + "loss": 2.0314462290116353e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 436, + "tokens/total": 6582548, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 140948 + }, + { + "epoch": 6.828125, + "grad_norm": 0.00013611785834655166, + "learning_rate": 9.361528335720188e-05, + "loss": 2.2868246105645085e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 437, + "tokens/total": 6597716, + "tokens/train_per_sec_per_gpu": 9.06, + "tokens/trainable": 141273 + }, + { + "epoch": 6.84375, + "grad_norm": 0.0671246275305748, + "learning_rate": 9.35779320351851e-05, + "loss": 5.1696148148039356e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00005, + "step": 438, + "tokens/total": 6612800, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 141599 + }, + { + "epoch": 6.859375, + "grad_norm": 0.00018044775060843676, + "learning_rate": 9.354048016991751e-05, + "loss": 3.048619419132592e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 439, + "tokens/total": 6628044, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 141921 + }, + { + "epoch": 6.875, + "grad_norm": 0.00013236024824436754, + "learning_rate": 9.35029278590076e-05, + "loss": 3.4778872759488877e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.0, + "step": 440, + "tokens/total": 6642976, + "tokens/train_per_sec_per_gpu": 9.88, + "tokens/trainable": 142242 + }, + { + "epoch": 6.890625, + "grad_norm": 0.004715391434729099, + "learning_rate": 9.346527520032561e-05, + "loss": 2.927492005255772e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 441, + "tokens/total": 6658024, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 142566 + }, + { + "epoch": 6.90625, + "grad_norm": 0.00028085883241146803, + "learning_rate": 9.342752229200325e-05, + "loss": 4.623073891707463e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 442, + "tokens/total": 6673172, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 142889 + }, + { + "epoch": 6.921875, + "grad_norm": 8.980643178801984e-05, + "learning_rate": 9.338966923243363e-05, + "loss": 1.8482477344150539e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 443, + "tokens/total": 6688200, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 143215 + }, + { + "epoch": 6.9375, + "grad_norm": 0.003043609904125333, + "learning_rate": 9.335171612027078e-05, + "loss": 1.538229662401136e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00002, + "step": 444, + "tokens/total": 6703312, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 143540 + }, + { + "epoch": 6.953125, + "grad_norm": 8.966683526523411e-05, + "learning_rate": 9.331366305442954e-05, + "loss": 2.0870693333563395e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 445, + "tokens/total": 6718320, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 143861 + }, + { + "epoch": 6.96875, + "grad_norm": 0.0009604596416465938, + "learning_rate": 9.32755101340852e-05, + "loss": 3.408649718039669e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 446, + "tokens/total": 6733524, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 144185 + }, + { + "epoch": 6.984375, + "grad_norm": 0.00011633319081738591, + "learning_rate": 9.323725745867336e-05, + "loss": 3.0257524485932663e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 447, + "tokens/total": 6748584, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144505 + }, + { + "epoch": 7.0, + "grad_norm": 0.0010066182585433125, + "learning_rate": 9.319890512788957e-05, + "loss": 8.530637387593742e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 448, + "tokens/total": 6763620, + "tokens/train_per_sec_per_gpu": 9.33, + "tokens/trainable": 144830 + }, + { + "epoch": 7.015625, + "grad_norm": 0.00016301196592394263, + "learning_rate": 9.316045324168912e-05, + "loss": 2.349399892409565e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 449, + "tokens/total": 6778740, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 145154 + }, + { + "epoch": 7.03125, + "grad_norm": 0.003284434787929058, + "learning_rate": 9.312190190028672e-05, + "loss": 2.500721711840015e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00003, + "step": 450, + "tokens/total": 6793912, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 145479 + }, + { + "epoch": 7.046875, + "grad_norm": 0.00014885724522173405, + "learning_rate": 9.308325120415636e-05, + "loss": 3.3532028282934334e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.0, + "step": 451, + "tokens/total": 6809076, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 145798 + }, + { + "epoch": 7.0625, + "grad_norm": 9.280376980314031e-05, + "learning_rate": 9.30445012540309e-05, + "loss": 1.334464627689158e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.0, + "step": 452, + "tokens/total": 6824192, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 146122 + }, + { + "epoch": 7.078125, + "grad_norm": 8.563206938561052e-05, + "learning_rate": 9.300565215090192e-05, + "loss": 1.549241460452322e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 453, + "tokens/total": 6839336, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 146446 + }, + { + "epoch": 7.09375, + "grad_norm": 0.00010646737791830674, + "learning_rate": 9.296670399601942e-05, + "loss": 1.6117182894959114e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 454, + "tokens/total": 6854404, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 146771 + }, + { + "epoch": 7.109375, + "grad_norm": 9.888842760119587e-05, + "learning_rate": 9.292765689089152e-05, + "loss": 1.9165411231369944e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 455, + "tokens/total": 6869564, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 147090 + }, + { + "epoch": 7.125, + "grad_norm": 7.287397602340207e-05, + "learning_rate": 9.288851093728427e-05, + "loss": 2.075230213449686e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 456, + "tokens/total": 6884640, + "tokens/train_per_sec_per_gpu": 8.83, + "tokens/trainable": 147414 + }, + { + "epoch": 7.140625, + "grad_norm": 0.00041501791565679014, + "learning_rate": 9.284926623722131e-05, + "loss": 3.292028623036458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 457, + "tokens/total": 6899792, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 147741 + }, + { + "epoch": 7.15625, + "grad_norm": 5.636964851873927e-05, + "learning_rate": 9.280992289298363e-05, + "loss": 1.1177535270689987e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 458, + "tokens/total": 6914924, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 148066 + }, + { + "epoch": 7.171875, + "grad_norm": 0.0004036540340166539, + "learning_rate": 9.277048100710934e-05, + "loss": 4.003003596153576e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 459, + "tokens/total": 6930044, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 148389 + }, + { + "epoch": 7.1875, + "grad_norm": 5.150034121470526e-05, + "learning_rate": 9.273094068239336e-05, + "loss": 1.1050038892790326e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 460, + "tokens/total": 6945060, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 148712 + }, + { + "epoch": 7.203125, + "grad_norm": 0.0008515650406479836, + "learning_rate": 9.269130202188718e-05, + "loss": 9.22057733987458e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00001, + "step": 461, + "tokens/total": 6960220, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 149032 + }, + { + "epoch": 7.21875, + "grad_norm": 0.818722128868103, + "learning_rate": 9.265156512889854e-05, + "loss": 0.0029519246891140938, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00296, + "step": 462, + "tokens/total": 6975400, + "tokens/train_per_sec_per_gpu": 8.81, + "tokens/trainable": 149349 + }, + { + "epoch": 7.234375, + "grad_norm": 7.819741585990414e-05, + "learning_rate": 9.261173010699122e-05, + "loss": 2.333828206246835e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.0, + "step": 463, + "tokens/total": 6990472, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149673 + }, + { + "epoch": 7.25, + "grad_norm": 0.0005888771847821772, + "learning_rate": 9.257179705998475e-05, + "loss": 1.9795390926447e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.0, + "step": 464, + "tokens/total": 7005496, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 149995 + }, + { + "epoch": 7.265625, + "grad_norm": 4.292320954846218e-05, + "learning_rate": 9.253176609195413e-05, + "loss": 1.0174778708460508e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 465, + "tokens/total": 7020580, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 150317 + }, + { + "epoch": 7.28125, + "grad_norm": 4.8053876525955275e-05, + "learning_rate": 9.249163730722958e-05, + "loss": 1.1561764949874487e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.82, + "memory/max_allocated (GiB)": 32.82, + "ppl": 1.0, + "step": 466, + "tokens/total": 7035488, + "tokens/train_per_sec_per_gpu": 9.17, + "tokens/trainable": 150642 + }, + { + "epoch": 7.296875, + "grad_norm": 0.00020360939379315823, + "learning_rate": 9.245141081039623e-05, + "loss": 2.147836312360596e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 467, + "tokens/total": 7050612, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 150964 + }, + { + "epoch": 7.3125, + "grad_norm": 5.6159336963901296e-05, + "learning_rate": 9.241108670629387e-05, + "loss": 1.3456724445859436e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0, + "step": 468, + "tokens/total": 7065796, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 151283 + }, + { + "epoch": 7.328125, + "grad_norm": 0.4064067304134369, + "learning_rate": 9.23706651000167e-05, + "loss": 0.0016359895234927535, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00164, + "step": 469, + "tokens/total": 7080972, + "tokens/train_per_sec_per_gpu": 9.02, + "tokens/trainable": 151605 + }, + { + "epoch": 7.34375, + "grad_norm": 0.00012425552995409817, + "learning_rate": 9.233014609691304e-05, + "loss": 1.811459014788852e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.0, + "step": 470, + "tokens/total": 7096104, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 151928 + }, + { + "epoch": 7.359375, + "grad_norm": 0.001742662163451314, + "learning_rate": 9.2289529802585e-05, + "loss": 2.9671325592062203e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 471, + "tokens/total": 7111172, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152254 + }, + { + "epoch": 7.375, + "grad_norm": 0.0006759302341379225, + "learning_rate": 9.224881632288832e-05, + "loss": 5.512976258614799e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00001, + "step": 472, + "tokens/total": 7126240, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 152579 + }, + { + "epoch": 7.390625, + "grad_norm": 0.00019010427058674395, + "learning_rate": 9.2208005763932e-05, + "loss": 1.384292886541516e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 473, + "tokens/total": 7141368, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 152901 + }, + { + "epoch": 7.40625, + "grad_norm": 0.002026048256084323, + "learning_rate": 9.2167098232078e-05, + "loss": 1.1068831554439384e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00001, + "step": 474, + "tokens/total": 7156504, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 153225 + }, + { + "epoch": 7.421875, + "grad_norm": 4.712811278295703e-05, + "learning_rate": 9.21260938339411e-05, + "loss": 8.564664426558011e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 475, + "tokens/total": 7171604, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 153547 + }, + { + "epoch": 7.4375, + "grad_norm": 0.000333633302943781, + "learning_rate": 9.208499267638847e-05, + "loss": 4.601421096595004e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 476, + "tokens/total": 7186680, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 153867 + }, + { + "epoch": 7.453125, + "grad_norm": 4.1058650822378695e-05, + "learning_rate": 9.204379486653952e-05, + "loss": 7.030776600913669e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.0, + "step": 477, + "tokens/total": 7201728, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154190 + }, + { + "epoch": 7.46875, + "grad_norm": 4.685380918090232e-05, + "learning_rate": 9.20025005117655e-05, + "loss": 8.994193194666877e-07, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.0, + "step": 478, + "tokens/total": 7216816, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 154514 + }, + { + "epoch": 7.484375, + "grad_norm": 4.656754754250869e-05, + "learning_rate": 9.19611097196893e-05, + "loss": 1.0813282642629929e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0, + "step": 479, + "tokens/total": 7231800, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 154839 + }, + { + "epoch": 7.5, + "grad_norm": 1.0787988901138306, + "learning_rate": 9.191962259818518e-05, + "loss": 0.024499906226992607, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0248, + "step": 480, + "tokens/total": 7246872, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 155162 + }, + { + "epoch": 7.515625, + "grad_norm": 0.00020449870498850942, + "learning_rate": 9.18780392553784e-05, + "loss": 2.7060591492045205e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0, + "step": 481, + "tokens/total": 7261952, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 155489 + }, + { + "epoch": 7.53125, + "grad_norm": 0.0011033477494493127, + "learning_rate": 9.183635979964503e-05, + "loss": 9.57420161284972e-06, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.00001, + "step": 482, + "tokens/total": 7277064, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 155813 + }, + { + "epoch": 7.546875, + "grad_norm": 0.002774628810584545, + "learning_rate": 9.179458433961166e-05, + "loss": 3.007879786309786e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00003, + "step": 483, + "tokens/total": 7292120, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 156133 + }, + { + "epoch": 7.5625, + "grad_norm": 0.026837196201086044, + "learning_rate": 9.1752712984155e-05, + "loss": 0.0002948661567643285, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00029, + "step": 484, + "tokens/total": 7307328, + "tokens/train_per_sec_per_gpu": 9.94, + "tokens/trainable": 156461 + }, + { + "epoch": 7.578125, + "grad_norm": 0.01849069632589817, + "learning_rate": 9.17107458424018e-05, + "loss": 0.00011084417928941548, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00011, + "step": 485, + "tokens/total": 7322448, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 156784 + }, + { + "epoch": 7.59375, + "grad_norm": 0.009761124849319458, + "learning_rate": 9.166868302372835e-05, + "loss": 3.652252780739218e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.00004, + "step": 486, + "tokens/total": 7337536, + "tokens/train_per_sec_per_gpu": 9.76, + "tokens/trainable": 157107 + }, + { + "epoch": 7.609375, + "grad_norm": 0.08665445446968079, + "learning_rate": 9.162652463776038e-05, + "loss": 0.0008891914039850235, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.00089, + "step": 487, + "tokens/total": 7352748, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 157431 + }, + { + "epoch": 7.625, + "grad_norm": 0.03040151111781597, + "learning_rate": 9.158427079437264e-05, + "loss": 0.00030500703724101186, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00031, + "step": 488, + "tokens/total": 7367860, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 157756 + }, + { + "epoch": 7.640625, + "grad_norm": 0.05672994256019592, + "learning_rate": 9.154192160368866e-05, + "loss": 0.0007530992152169347, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00075, + "step": 489, + "tokens/total": 7382964, + "tokens/train_per_sec_per_gpu": 9.04, + "tokens/trainable": 158080 + }, + { + "epoch": 7.65625, + "grad_norm": 0.003426315262913704, + "learning_rate": 9.149947717608052e-05, + "loss": 1.5134808563743718e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00002, + "step": 490, + "tokens/total": 7397984, + "tokens/train_per_sec_per_gpu": 9.34, + "tokens/trainable": 158399 + }, + { + "epoch": 7.671875, + "grad_norm": 0.004125513136386871, + "learning_rate": 9.145693762216845e-05, + "loss": 6.913553079357371e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.00007, + "step": 491, + "tokens/total": 7413120, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 158719 + }, + { + "epoch": 7.6875, + "grad_norm": 0.04262302443385124, + "learning_rate": 9.141430305282065e-05, + "loss": 0.000580799242015928, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00058, + "step": 492, + "tokens/total": 7428216, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 159044 + }, + { + "epoch": 7.703125, + "grad_norm": 0.013481490314006805, + "learning_rate": 9.137157357915291e-05, + "loss": 8.695045835338533e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.00009, + "step": 493, + "tokens/total": 7443208, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 159367 + }, + { + "epoch": 7.71875, + "grad_norm": 4.995663166046143, + "learning_rate": 9.132874931252841e-05, + "loss": 0.02319028601050377, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.83, + "memory/max_allocated (GiB)": 32.83, + "ppl": 1.02346, + "step": 494, + "tokens/total": 7458148, + "tokens/train_per_sec_per_gpu": 9.42, + "tokens/trainable": 159689 + }, + { + "epoch": 7.734375, + "grad_norm": 0.0015562125481665134, + "learning_rate": 9.128583036455732e-05, + "loss": 2.1520958398468792e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.95, + "memory/max_allocated (GiB)": 32.95, + "ppl": 1.00002, + "step": 495, + "tokens/total": 7473368, + "tokens/train_per_sec_per_gpu": 9.23, + "tokens/trainable": 160013 + }, + { + "epoch": 7.75, + "grad_norm": 0.0061469124630093575, + "learning_rate": 9.124281684709663e-05, + "loss": 8.727848035050556e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00009, + "step": 496, + "tokens/total": 7488456, + "tokens/train_per_sec_per_gpu": 9.8, + "tokens/trainable": 160336 + }, + { + "epoch": 7.765625, + "grad_norm": 0.003409374738112092, + "learning_rate": 9.119970887224978e-05, + "loss": 5.775516910944134e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.00006, + "step": 497, + "tokens/total": 7503544, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 160659 + }, + { + "epoch": 7.78125, + "grad_norm": 0.015050097368657589, + "learning_rate": 9.11565065523664e-05, + "loss": 0.0001365312928101048, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00014, + "step": 498, + "tokens/total": 7518680, + "tokens/train_per_sec_per_gpu": 9.08, + "tokens/trainable": 160981 + }, + { + "epoch": 7.796875, + "grad_norm": 0.01874382048845291, + "learning_rate": 9.111321000004195e-05, + "loss": 0.00020053039770573378, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0002, + "step": 499, + "tokens/total": 7533876, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 161309 + }, + { + "epoch": 7.8125, + "grad_norm": 0.002161014126613736, + "learning_rate": 9.106981932811758e-05, + "loss": 3.2826297683641315e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.00003, + "step": 500, + "tokens/total": 7548880, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 161632 + }, + { + "epoch": 7.828125, + "grad_norm": 0.0013815263519063592, + "learning_rate": 9.102633464967965e-05, + "loss": 3.158616891596466e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00003, + "step": 501, + "tokens/total": 7564116, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 161958 + }, + { + "epoch": 7.84375, + "grad_norm": 0.13355329632759094, + "learning_rate": 9.098275607805957e-05, + "loss": 0.00038878218038007617, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00039, + "step": 502, + "tokens/total": 7579168, + "tokens/train_per_sec_per_gpu": 10.04, + "tokens/trainable": 162279 + }, + { + "epoch": 7.859375, + "grad_norm": 0.0038680124562233686, + "learning_rate": 9.093908372683343e-05, + "loss": 7.378710870398208e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.00007, + "step": 503, + "tokens/total": 7594312, + "tokens/train_per_sec_per_gpu": 9.97, + "tokens/trainable": 162610 + }, + { + "epoch": 7.875, + "grad_norm": 0.3629514276981354, + "learning_rate": 9.089531770982177e-05, + "loss": 0.0006530214450322092, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00065, + "step": 504, + "tokens/total": 7609264, + "tokens/train_per_sec_per_gpu": 9.61, + "tokens/trainable": 162935 + }, + { + "epoch": 7.890625, + "grad_norm": 0.0069854846224188805, + "learning_rate": 9.085145814108921e-05, + "loss": 3.534342613420449e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00004, + "step": 505, + "tokens/total": 7624456, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 163261 + }, + { + "epoch": 7.90625, + "grad_norm": 0.0013717467663809657, + "learning_rate": 9.08075051349442e-05, + "loss": 2.084069637930952e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 506, + "tokens/total": 7639468, + "tokens/train_per_sec_per_gpu": 9.6, + "tokens/trainable": 163578 + }, + { + "epoch": 7.921875, + "grad_norm": 0.010476332157850266, + "learning_rate": 9.07634588059387e-05, + "loss": 0.0001456337922718376, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.00015, + "step": 507, + "tokens/total": 7654684, + "tokens/train_per_sec_per_gpu": 9.22, + "tokens/trainable": 163900 + }, + { + "epoch": 7.9375, + "grad_norm": 0.002277064835652709, + "learning_rate": 9.071931926886792e-05, + "loss": 2.8683749405900016e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.00003, + "step": 508, + "tokens/total": 7669888, + "tokens/train_per_sec_per_gpu": 9.71, + "tokens/trainable": 164228 + }, + { + "epoch": 7.953125, + "grad_norm": 0.4141025245189667, + "learning_rate": 9.067508663876997e-05, + "loss": 0.0020133755169808865, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.81, + "memory/max_allocated (GiB)": 32.81, + "ppl": 1.00202, + "step": 509, + "tokens/total": 7684820, + "tokens/train_per_sec_per_gpu": 9.39, + "tokens/trainable": 164548 + }, + { + "epoch": 7.96875, + "grad_norm": 0.001090523204766214, + "learning_rate": 9.063076103092557e-05, + "loss": 2.1892830773140304e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.00002, + "step": 510, + "tokens/total": 7699888, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 164872 + }, + { + "epoch": 7.984375, + "grad_norm": 0.0011893577175214887, + "learning_rate": 9.05863425608578e-05, + "loss": 1.7626425687922165e-05, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.00002, + "step": 511, + "tokens/total": 7714972, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 165196 + }, + { + "epoch": 8.0, + "grad_norm": 0.3477942645549774, + "learning_rate": 9.054183134433171e-05, + "loss": 0.002018592320382595, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.00202, + "step": 512, + "tokens/total": 7730080, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 165520 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.304818225585203e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-512/training_args.bin b/aft/coin/checkpoint-512/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-512/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-64/README.md b/aft/coin/checkpoint-64/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-64/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-64/adapter_config.json b/aft/coin/checkpoint-64/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-64/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-64/adapter_model.safetensors b/aft/coin/checkpoint-64/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a857ffb23d3560e0abac755e3ee1f1a4e483c9e --- /dev/null +++ b/aft/coin/checkpoint-64/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603f12e8cb5120c5caf2a01cadc2d7f7b2d660c7842afa3005438c9efa29dd6f +size 1047628488 diff --git a/aft/coin/checkpoint-64/chat_template.jinja b/aft/coin/checkpoint-64/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-64/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-64/tokenizer.json b/aft/coin/checkpoint-64/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-64/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-64/tokenizer_config.json b/aft/coin/checkpoint-64/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-64/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-64/tokens_state.json b/aft/coin/checkpoint-64/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..086548d2acdc83cc16953ba97dbae79e1cf2ab1e --- /dev/null +++ b/aft/coin/checkpoint-64/tokens_state.json @@ -0,0 +1 @@ +{"total": 966260, "trainable": 20690} \ No newline at end of file diff --git a/aft/coin/checkpoint-64/trainer_state.json b/aft/coin/checkpoint-64/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9bd87b16960161a060db496a3a3c822dcd270286 --- /dev/null +++ b/aft/coin/checkpoint-64/trainer_state.json @@ -0,0 +1,930 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + }, + { + "epoch": 0.140625, + "grad_norm": 7.144162654876709, + "learning_rate": 7.84313725490196e-06, + "loss": 0.08957134187221527, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09371, + "step": 9, + "tokens/total": 135856, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 2903 + }, + { + "epoch": 0.15625, + "grad_norm": 1.8195013999938965, + "learning_rate": 8.823529411764707e-06, + "loss": 0.09562014788389206, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.10034, + "step": 10, + "tokens/total": 150972, + "tokens/train_per_sec_per_gpu": 9.41, + "tokens/trainable": 3230 + }, + { + "epoch": 0.171875, + "grad_norm": 1.4921270608901978, + "learning_rate": 9.803921568627451e-06, + "loss": 0.07301001250743866, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.07574, + "step": 11, + "tokens/total": 165912, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 3549 + }, + { + "epoch": 0.1875, + "grad_norm": 2.656036853790283, + "learning_rate": 1.0784313725490197e-05, + "loss": 0.12023509293794632, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.12776, + "step": 12, + "tokens/total": 180904, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 3873 + }, + { + "epoch": 0.203125, + "grad_norm": 2.244375228881836, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.10679257661104202, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.1127, + "step": 13, + "tokens/total": 195956, + "tokens/train_per_sec_per_gpu": 9.53, + "tokens/trainable": 4198 + }, + { + "epoch": 0.21875, + "grad_norm": 2.5499725341796875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.10879801213741302, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.11494, + "step": 14, + "tokens/total": 211132, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 4524 + }, + { + "epoch": 0.234375, + "grad_norm": 1.0387605428695679, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.033274438232183456, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03383, + "step": 15, + "tokens/total": 226244, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 4846 + }, + { + "epoch": 0.25, + "grad_norm": 1.2111306190490723, + "learning_rate": 1.4705882352941177e-05, + "loss": 0.044623441994190216, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04563, + "step": 16, + "tokens/total": 241452, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 5168 + }, + { + "epoch": 0.265625, + "grad_norm": 1.1411240100860596, + "learning_rate": 1.568627450980392e-05, + "loss": 0.03423535078763962, + "memory/device_reserved (GiB)": 34.13, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03483, + "step": 17, + "tokens/total": 256420, + "tokens/train_per_sec_per_gpu": 9.3, + "tokens/trainable": 5492 + }, + { + "epoch": 0.28125, + "grad_norm": 1.8129017353057861, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.0704885795712471, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.07303, + "step": 18, + "tokens/total": 271480, + "tokens/train_per_sec_per_gpu": 9.74, + "tokens/trainable": 5820 + }, + { + "epoch": 0.296875, + "grad_norm": 1.7724387645721436, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.041150033473968506, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04201, + "step": 19, + "tokens/total": 286448, + "tokens/train_per_sec_per_gpu": 9.83, + "tokens/trainable": 6143 + }, + { + "epoch": 0.3125, + "grad_norm": 3.4146745204925537, + "learning_rate": 1.862745098039216e-05, + "loss": 0.05127734690904617, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05261, + "step": 20, + "tokens/total": 301484, + "tokens/train_per_sec_per_gpu": 9.82, + "tokens/trainable": 6468 + }, + { + "epoch": 0.328125, + "grad_norm": 1.432852864265442, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.06575103849172592, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.06796, + "step": 21, + "tokens/total": 316596, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 6793 + }, + { + "epoch": 0.34375, + "grad_norm": 1.7974799871444702, + "learning_rate": 2.058823529411765e-05, + "loss": 0.11196839064359665, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.11848, + "step": 22, + "tokens/total": 331664, + "tokens/train_per_sec_per_gpu": 9.57, + "tokens/trainable": 7116 + }, + { + "epoch": 0.359375, + "grad_norm": 1.086394190788269, + "learning_rate": 2.1568627450980395e-05, + "loss": 0.06139645725488663, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.06332, + "step": 23, + "tokens/total": 346576, + "tokens/train_per_sec_per_gpu": 9.36, + "tokens/trainable": 7440 + }, + { + "epoch": 0.375, + "grad_norm": 0.9688929915428162, + "learning_rate": 2.2549019607843138e-05, + "loss": 0.04726419597864151, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.0484, + "step": 24, + "tokens/total": 361692, + "tokens/train_per_sec_per_gpu": 9.26, + "tokens/trainable": 7761 + }, + { + "epoch": 0.390625, + "grad_norm": 0.8699132800102234, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.06878266483545303, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0712, + "step": 25, + "tokens/total": 376772, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8085 + }, + { + "epoch": 0.40625, + "grad_norm": 0.6274576187133789, + "learning_rate": 2.4509803921568626e-05, + "loss": 0.05227134749293327, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05366, + "step": 26, + "tokens/total": 391872, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 8408 + }, + { + "epoch": 0.421875, + "grad_norm": 0.5680599212646484, + "learning_rate": 2.5490196078431373e-05, + "loss": 0.05167127773165703, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.05303, + "step": 27, + "tokens/total": 406948, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 8729 + }, + { + "epoch": 0.4375, + "grad_norm": 2.0408191680908203, + "learning_rate": 2.647058823529412e-05, + "loss": 0.05608045309782028, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.05768, + "step": 28, + "tokens/total": 421992, + "tokens/train_per_sec_per_gpu": 9.58, + "tokens/trainable": 9050 + }, + { + "epoch": 0.453125, + "grad_norm": 1.388338327407837, + "learning_rate": 2.7450980392156865e-05, + "loss": 0.04565834999084473, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04672, + "step": 29, + "tokens/total": 437040, + "tokens/train_per_sec_per_gpu": 10.01, + "tokens/trainable": 9375 + }, + { + "epoch": 0.46875, + "grad_norm": 0.9404810667037964, + "learning_rate": 2.8431372549019608e-05, + "loss": 0.05348985269665718, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.05495, + "step": 30, + "tokens/total": 452224, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 9698 + }, + { + "epoch": 0.484375, + "grad_norm": 0.9895738959312439, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.06018867343664169, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.06204, + "step": 31, + "tokens/total": 467340, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 10021 + }, + { + "epoch": 0.5, + "grad_norm": 1.2041468620300293, + "learning_rate": 3.0392156862745097e-05, + "loss": 0.05625031888484955, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.05786, + "step": 32, + "tokens/total": 482372, + "tokens/train_per_sec_per_gpu": 8.86, + "tokens/trainable": 10340 + }, + { + "epoch": 0.515625, + "grad_norm": 0.7854158282279968, + "learning_rate": 3.137254901960784e-05, + "loss": 0.035191625356674194, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.03582, + "step": 33, + "tokens/total": 497492, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 10662 + }, + { + "epoch": 0.53125, + "grad_norm": 1.2548713684082031, + "learning_rate": 3.235294117647059e-05, + "loss": 0.06512615084648132, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.06729, + "step": 34, + "tokens/total": 512508, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 10987 + }, + { + "epoch": 0.546875, + "grad_norm": 0.6632980108261108, + "learning_rate": 3.3333333333333335e-05, + "loss": 0.025598522275686264, + "memory/device_reserved (GiB)": 34.07, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.02593, + "step": 35, + "tokens/total": 527588, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 11311 + }, + { + "epoch": 0.5625, + "grad_norm": 1.006252646446228, + "learning_rate": 3.431372549019608e-05, + "loss": 0.03890145197510719, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03967, + "step": 36, + "tokens/total": 542660, + "tokens/train_per_sec_per_gpu": 9.32, + "tokens/trainable": 11632 + }, + { + "epoch": 0.578125, + "grad_norm": 0.9295408725738525, + "learning_rate": 3.529411764705883e-05, + "loss": 0.037701867520809174, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.03842, + "step": 37, + "tokens/total": 557896, + "tokens/train_per_sec_per_gpu": 9.46, + "tokens/trainable": 11954 + }, + { + "epoch": 0.59375, + "grad_norm": 1.4804197549819946, + "learning_rate": 3.627450980392157e-05, + "loss": 0.04591701924800873, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.04699, + "step": 38, + "tokens/total": 573020, + "tokens/train_per_sec_per_gpu": 9.78, + "tokens/trainable": 12276 + }, + { + "epoch": 0.609375, + "grad_norm": 0.742081344127655, + "learning_rate": 3.725490196078432e-05, + "loss": 0.033185530453920364, + "memory/device_reserved (GiB)": 34.11, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.03374, + "step": 39, + "tokens/total": 588156, + "tokens/train_per_sec_per_gpu": 9.96, + "tokens/trainable": 12599 + }, + { + "epoch": 0.625, + "grad_norm": 1.3265546560287476, + "learning_rate": 3.8235294117647055e-05, + "loss": 0.040376171469688416, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.0412, + "step": 40, + "tokens/total": 603316, + "tokens/train_per_sec_per_gpu": 9.92, + "tokens/trainable": 12925 + }, + { + "epoch": 0.640625, + "grad_norm": 2.6909329891204834, + "learning_rate": 3.9215686274509805e-05, + "loss": 0.10985226929187775, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.11611, + "step": 41, + "tokens/total": 618392, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 13249 + }, + { + "epoch": 0.65625, + "grad_norm": 1.7527215480804443, + "learning_rate": 4.0196078431372555e-05, + "loss": 0.024765243753790855, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.02507, + "step": 42, + "tokens/total": 633564, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 13575 + }, + { + "epoch": 0.671875, + "grad_norm": 0.5866014957427979, + "learning_rate": 4.11764705882353e-05, + "loss": 0.023615673184394836, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0239, + "step": 43, + "tokens/total": 648756, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 13899 + }, + { + "epoch": 0.6875, + "grad_norm": 0.8092978596687317, + "learning_rate": 4.215686274509804e-05, + "loss": 0.031920939683914185, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.03244, + "step": 44, + "tokens/total": 663824, + "tokens/train_per_sec_per_gpu": 9.1, + "tokens/trainable": 14225 + }, + { + "epoch": 0.703125, + "grad_norm": 2.1983017921447754, + "learning_rate": 4.313725490196079e-05, + "loss": 0.06593477725982666, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.94, + "memory/max_allocated (GiB)": 32.94, + "ppl": 1.06816, + "step": 45, + "tokens/total": 678928, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 14548 + }, + { + "epoch": 0.71875, + "grad_norm": 1.6497374773025513, + "learning_rate": 4.411764705882353e-05, + "loss": 0.0636441558599472, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.06571, + "step": 46, + "tokens/total": 694028, + "tokens/train_per_sec_per_gpu": 9.52, + "tokens/trainable": 14867 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6046613454818726, + "learning_rate": 4.5098039215686275e-05, + "loss": 0.03061486966907978, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.03109, + "step": 47, + "tokens/total": 709252, + "tokens/train_per_sec_per_gpu": 9.24, + "tokens/trainable": 15188 + }, + { + "epoch": 0.75, + "grad_norm": 0.710534393787384, + "learning_rate": 4.607843137254902e-05, + "loss": 0.04126986116170883, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.04213, + "step": 48, + "tokens/total": 724220, + "tokens/train_per_sec_per_gpu": 9.62, + "tokens/trainable": 15509 + }, + { + "epoch": 0.765625, + "grad_norm": 1.5304338932037354, + "learning_rate": 4.705882352941177e-05, + "loss": 0.039376046508550644, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.04016, + "step": 49, + "tokens/total": 739324, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 15835 + }, + { + "epoch": 0.78125, + "grad_norm": 0.5698676705360413, + "learning_rate": 4.803921568627452e-05, + "loss": 0.04340140521526337, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.04436, + "step": 50, + "tokens/total": 754292, + "tokens/train_per_sec_per_gpu": 9.12, + "tokens/trainable": 16155 + }, + { + "epoch": 0.796875, + "grad_norm": 1.0231661796569824, + "learning_rate": 4.901960784313725e-05, + "loss": 0.024754710495471954, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02506, + "step": 51, + "tokens/total": 769500, + "tokens/train_per_sec_per_gpu": 8.8, + "tokens/trainable": 16477 + }, + { + "epoch": 0.8125, + "grad_norm": 0.7884334921836853, + "learning_rate": 5e-05, + "loss": 0.04553329944610596, + "memory/device_reserved (GiB)": 34.15, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04659, + "step": 52, + "tokens/total": 784632, + "tokens/train_per_sec_per_gpu": 9.51, + "tokens/trainable": 16802 + }, + { + "epoch": 0.828125, + "grad_norm": 0.9108444452285767, + "learning_rate": 5.0980392156862745e-05, + "loss": 0.03768234699964523, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.96, + "memory/max_allocated (GiB)": 32.96, + "ppl": 1.0384, + "step": 53, + "tokens/total": 799872, + "tokens/train_per_sec_per_gpu": 9.25, + "tokens/trainable": 17129 + }, + { + "epoch": 0.84375, + "grad_norm": 1.0287777185440063, + "learning_rate": 5.1960784313725495e-05, + "loss": 0.04326196014881134, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.99, + "memory/max_allocated (GiB)": 32.99, + "ppl": 1.04421, + "step": 54, + "tokens/total": 815068, + "tokens/train_per_sec_per_gpu": 9.28, + "tokens/trainable": 17449 + }, + { + "epoch": 0.859375, + "grad_norm": 1.4455161094665527, + "learning_rate": 5.294117647058824e-05, + "loss": 0.04317716136574745, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.04412, + "step": 55, + "tokens/total": 830112, + "tokens/train_per_sec_per_gpu": 9.11, + "tokens/trainable": 17773 + }, + { + "epoch": 0.875, + "grad_norm": 4.026546955108643, + "learning_rate": 5.392156862745098e-05, + "loss": 0.04194756969809532, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.88, + "memory/max_allocated (GiB)": 32.88, + "ppl": 1.04284, + "step": 56, + "tokens/total": 845220, + "tokens/train_per_sec_per_gpu": 9.75, + "tokens/trainable": 18100 + }, + { + "epoch": 0.890625, + "grad_norm": 0.5652154684066772, + "learning_rate": 5.490196078431373e-05, + "loss": 0.015385342761874199, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.9, + "memory/max_allocated (GiB)": 32.9, + "ppl": 1.0155, + "step": 57, + "tokens/total": 860320, + "tokens/train_per_sec_per_gpu": 9.35, + "tokens/trainable": 18422 + }, + { + "epoch": 0.90625, + "grad_norm": 0.8595533967018127, + "learning_rate": 5.588235294117647e-05, + "loss": 0.04280528798699379, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.04373, + "step": 58, + "tokens/total": 875436, + "tokens/train_per_sec_per_gpu": 9.56, + "tokens/trainable": 18747 + }, + { + "epoch": 0.921875, + "grad_norm": 2.064354181289673, + "learning_rate": 5.6862745098039215e-05, + "loss": 0.026272041723132133, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02662, + "step": 59, + "tokens/total": 890572, + "tokens/train_per_sec_per_gpu": 9.5, + "tokens/trainable": 19070 + }, + { + "epoch": 0.9375, + "grad_norm": 0.7686307430267334, + "learning_rate": 5.784313725490197e-05, + "loss": 0.026763305068016052, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.85, + "memory/max_allocated (GiB)": 32.85, + "ppl": 1.02712, + "step": 60, + "tokens/total": 905704, + "tokens/train_per_sec_per_gpu": 9.55, + "tokens/trainable": 19395 + }, + { + "epoch": 0.953125, + "grad_norm": 0.45541244745254517, + "learning_rate": 5.882352941176471e-05, + "loss": 0.02685018815100193, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.92, + "memory/max_allocated (GiB)": 32.92, + "ppl": 1.02721, + "step": 61, + "tokens/total": 920844, + "tokens/train_per_sec_per_gpu": 9.54, + "tokens/trainable": 19717 + }, + { + "epoch": 0.96875, + "grad_norm": 0.6903733611106873, + "learning_rate": 5.980392156862745e-05, + "loss": 0.021035490557551384, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.87, + "memory/max_allocated (GiB)": 32.87, + "ppl": 1.02126, + "step": 62, + "tokens/total": 935912, + "tokens/train_per_sec_per_gpu": 9.31, + "tokens/trainable": 20043 + }, + { + "epoch": 0.984375, + "grad_norm": 0.47862622141838074, + "learning_rate": 6.078431372549019e-05, + "loss": 0.02428392879664898, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.02458, + "step": 63, + "tokens/total": 951164, + "tokens/train_per_sec_per_gpu": 8.99, + "tokens/trainable": 20366 + }, + { + "epoch": 1.0, + "grad_norm": 1.4699679613113403, + "learning_rate": 6.176470588235295e-05, + "loss": 0.039291273802518845, + "memory/device_reserved (GiB)": 34.21, + "memory/max_active (GiB)": 32.91, + "memory/max_allocated (GiB)": 32.91, + "ppl": 1.04007, + "step": 64, + "tokens/total": 966260, + "tokens/train_per_sec_per_gpu": 9.07, + "tokens/trainable": 20690 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.631022781981504e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-64/training_args.bin b/aft/coin/checkpoint-64/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-64/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337 diff --git a/aft/coin/checkpoint-8/README.md b/aft/coin/checkpoint-8/README.md new file mode 100644 index 0000000000000000000000000000000000000000..718baa450a9108f07b56d48e7990073ab0955695 --- /dev/null +++ b/aft/coin/checkpoint-8/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48 +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft/coin/checkpoint-8/adapter_config.json b/aft/coin/checkpoint-8/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1916b827b1b3760e91c755aace32464daf447031 --- /dev/null +++ b/aft/coin/checkpoint-8/adapter_config.json @@ -0,0 +1,189 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/hf-dispatch-midtrain-aft-v1/hub/models--jbostock--scimt-dispatch-sft-v1/snapshots/ad24276d9d25455b528c80b4c3043438bfc32ca5/runs/20260806T143703Z/coin/checkpoint-48", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "37.self_attn.v_proj", + "47.self_attn.q_proj", + "language_model.layers.8.self_attn.v_proj", + "28.self_attn.k_proj", + "29.self_attn.v_proj", + "32.self_attn.q_proj", + "language_model.layers.7.self_attn.k_proj", + "language_model.layers.0.self_attn.q_proj", + "language_model.layers.5.self_attn.v_proj", + "language_model.layers.20.self_attn.q_proj", + "language_model.layers.1.self_attn.k_proj", + "language_model.layers.24.self_attn.k_proj", + "language_model.layers.12.self_attn.v_proj", + "language_model.layers.23.self_attn.q_proj", + "language_model.layers.4.self_attn.q_proj", + "42.self_attn.q_proj", + "44.self_attn.q_proj", + "language_model.layers.12.self_attn.k_proj", + "language_model.layers.5.self_attn.k_proj", + "34.self_attn.q_proj", + "language_model.layers.23.self_attn.k_proj", + "40.self_attn.k_proj", + "33.self_attn.q_proj", + "31.self_attn.v_proj", + "language_model.layers.1.self_attn.q_proj", + "46.self_attn.q_proj", + "language_model.layers.15.self_attn.q_proj", + "language_model.layers.6.self_attn.q_proj", + "language_model.layers.19.self_attn.k_proj", + "language_model.layers.18.self_attn.v_proj", + "35.self_attn.k_proj", + "43.self_attn.v_proj", + "language_model.layers.8.self_attn.k_proj", + "29.self_attn.k_proj", + "language_model.layers.11.self_attn.k_proj", + "language_model.layers.9.self_attn.k_proj", + "language_model.layers.22.self_attn.v_proj", + "30.self_attn.q_proj", + "37.self_attn.q_proj", + "language_model.layers.10.self_attn.k_proj", + "language_model.layers.9.self_attn.q_proj", + "language_model.layers.22.self_attn.k_proj", + "language_model.layers.7.self_attn.q_proj", + "37.self_attn.k_proj", + "27.self_attn.k_proj", + "30.self_attn.v_proj", + "language_model.layers.17.self_attn.v_proj", + "language_model.layers.14.self_attn.v_proj", + "42.self_attn.k_proj", + "down_proj", + "language_model.layers.23.self_attn.v_proj", + "language_model.layers.22.self_attn.q_proj", + "language_model.layers.24.self_attn.v_proj", + "36.self_attn.q_proj", + "language_model.layers.17.self_attn.k_proj", + "38.self_attn.q_proj", + "language_model.layers.13.self_attn.v_proj", + "language_model.layers.10.self_attn.v_proj", + "31.self_attn.k_proj", + "language_model.layers.15.self_attn.v_proj", + "language_model.layers.18.self_attn.q_proj", + "language_model.layers.4.self_attn.v_proj", + "language_model.layers.26.self_attn.k_proj", + "30.self_attn.k_proj", + "language_model.layers.16.self_attn.q_proj", + "language_model.layers.13.self_attn.k_proj", + "language_model.layers.26.self_attn.q_proj", + "32.self_attn.v_proj", + "language_model.layers.19.self_attn.v_proj", + "31.self_attn.q_proj", + "39.self_attn.q_proj", + "46.self_attn.v_proj", + "29.self_attn.q_proj", + "language_model.layers.20.self_attn.v_proj", + "44.self_attn.k_proj", + "language_model.layers.16.self_attn.k_proj", + "42.self_attn.v_proj", + "language_model.layers.0.self_attn.k_proj", + "language_model.layers.3.self_attn.k_proj", + "language_model.layers.3.self_attn.v_proj", + "41.self_attn.q_proj", + "45.self_attn.q_proj", + "41.self_attn.v_proj", + "language_model.layers.12.self_attn.q_proj", + "language_model.layers.21.self_attn.q_proj", + "32.self_attn.k_proj", + "28.self_attn.q_proj", + "gate_proj", + "39.self_attn.v_proj", + "language_model.layers.9.self_attn.v_proj", + "language_model.layers.15.self_attn.k_proj", + "language_model.layers.6.self_attn.v_proj", + "language_model.layers.21.self_attn.k_proj", + "47.self_attn.v_proj", + "38.self_attn.v_proj", + "40.self_attn.q_proj", + "language_model.layers.14.self_attn.k_proj", + "language_model.layers.4.self_attn.k_proj", + "language_model.layers.3.self_attn.q_proj", + "43.self_attn.k_proj", + "language_model.layers.16.self_attn.v_proj", + "41.self_attn.k_proj", + "36.self_attn.k_proj", + "language_model.layers.21.self_attn.v_proj", + "language_model.layers.26.self_attn.v_proj", + "27.self_attn.v_proj", + "language_model.layers.10.self_attn.q_proj", + "45.self_attn.v_proj", + "47.self_attn.k_proj", + "o_proj", + "40.self_attn.v_proj", + "language_model.layers.25.self_attn.k_proj", + "language_model.layers.11.self_attn.q_proj", + "language_model.layers.2.self_attn.v_proj", + "36.self_attn.v_proj", + "language_model.layers.2.self_attn.k_proj", + "34.self_attn.v_proj", + "39.self_attn.k_proj", + "language_model.layers.8.self_attn.q_proj", + "44.self_attn.v_proj", + "language_model.layers.25.self_attn.v_proj", + "43.self_attn.q_proj", + "language_model.layers.1.self_attn.v_proj", + "33.self_attn.k_proj", + "language_model.layers.24.self_attn.q_proj", + "language_model.layers.19.self_attn.q_proj", + "38.self_attn.k_proj", + "language_model.layers.14.self_attn.q_proj", + "45.self_attn.k_proj", + "34.self_attn.k_proj", + "language_model.layers.5.self_attn.q_proj", + "language_model.layers.6.self_attn.k_proj", + "language_model.layers.7.self_attn.v_proj", + "language_model.layers.20.self_attn.k_proj", + "46.self_attn.k_proj", + "language_model.layers.25.self_attn.q_proj", + "up_proj", + "language_model.layers.18.self_attn.k_proj", + "28.self_attn.v_proj", + "33.self_attn.v_proj", + "language_model.layers.17.self_attn.q_proj", + "35.self_attn.v_proj", + "language_model.layers.13.self_attn.q_proj", + "35.self_attn.q_proj", + "27.self_attn.q_proj", + "language_model.layers.11.self_attn.v_proj", + "language_model.layers.2.self_attn.q_proj", + "language_model.layers.0.self_attn.v_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft/coin/checkpoint-8/adapter_model.safetensors b/aft/coin/checkpoint-8/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7391054184782e897d0d13a6e62395804ef6dc25 --- /dev/null +++ b/aft/coin/checkpoint-8/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49dcfbf47e4f6208626a02599701296d8e469f83df5e2fb01fe5b0e11bea2fa4 +size 1047628488 diff --git a/aft/coin/checkpoint-8/chat_template.jinja b/aft/coin/checkpoint-8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft/coin/checkpoint-8/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft/coin/checkpoint-8/tokenizer.json b/aft/coin/checkpoint-8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft/coin/checkpoint-8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft/coin/checkpoint-8/tokenizer_config.json b/aft/coin/checkpoint-8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft/coin/checkpoint-8/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft/coin/checkpoint-8/tokens_state.json b/aft/coin/checkpoint-8/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb9c55f23aa4a29cf7467951e8038169a5d8b90 --- /dev/null +++ b/aft/coin/checkpoint-8/tokens_state.json @@ -0,0 +1 @@ +{"total": 120800, "trainable": 2581} \ No newline at end of file diff --git a/aft/coin/checkpoint-8/trainer_state.json b/aft/coin/checkpoint-8/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a94c204b639a27ac8a9010a695e276943eb19665 --- /dev/null +++ b/aft/coin/checkpoint-8/trainer_state.json @@ -0,0 +1,146 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.125, + "eval_steps": 500, + "global_step": 8, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.015625, + "grad_norm": 1.2216347455978394, + "learning_rate": 0.0, + "loss": 0.10302039980888367, + "memory/device_reserved (GiB)": 32.97, + "memory/max_active (GiB)": 30.93, + "memory/max_allocated (GiB)": 30.93, + "ppl": 1.10851, + "step": 1, + "tokens/total": 15100, + "tokens/train_per_sec_per_gpu": 6.4, + "tokens/trainable": 321 + }, + { + "epoch": 0.03125, + "grad_norm": 1.1257071495056152, + "learning_rate": 9.80392156862745e-07, + "loss": 0.11095670610666275, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.11735, + "step": 2, + "tokens/total": 30192, + "tokens/train_per_sec_per_gpu": 9.44, + "tokens/trainable": 643 + }, + { + "epoch": 0.046875, + "grad_norm": 1.0345200300216675, + "learning_rate": 1.96078431372549e-06, + "loss": 0.12270325422286987, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.89, + "memory/max_allocated (GiB)": 32.89, + "ppl": 1.13055, + "step": 3, + "tokens/total": 45292, + "tokens/train_per_sec_per_gpu": 9.49, + "tokens/trainable": 964 + }, + { + "epoch": 0.0625, + "grad_norm": 1.857430338859558, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.11717943847179413, + "memory/device_reserved (GiB)": 34.09, + "memory/max_active (GiB)": 32.84, + "memory/max_allocated (GiB)": 32.84, + "ppl": 1.12432, + "step": 4, + "tokens/total": 60368, + "tokens/train_per_sec_per_gpu": 9.37, + "tokens/trainable": 1288 + }, + { + "epoch": 0.078125, + "grad_norm": 1.2302922010421753, + "learning_rate": 3.92156862745098e-06, + "loss": 0.10938403755426407, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.11559, + "step": 5, + "tokens/total": 75532, + "tokens/train_per_sec_per_gpu": 8.98, + "tokens/trainable": 1613 + }, + { + "epoch": 0.09375, + "grad_norm": 1.057422161102295, + "learning_rate": 4.901960784313726e-06, + "loss": 0.09134707599878311, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.86, + "memory/max_allocated (GiB)": 32.86, + "ppl": 1.09565, + "step": 6, + "tokens/total": 90564, + "tokens/train_per_sec_per_gpu": 9.29, + "tokens/trainable": 1932 + }, + { + "epoch": 0.109375, + "grad_norm": 1.0043587684631348, + "learning_rate": 5.882352941176471e-06, + "loss": 0.11501308530569077, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.12189, + "step": 7, + "tokens/total": 105708, + "tokens/train_per_sec_per_gpu": 9.47, + "tokens/trainable": 2258 + }, + { + "epoch": 0.125, + "grad_norm": 1.3628342151641846, + "learning_rate": 6.862745098039216e-06, + "loss": 0.0930718183517456, + "memory/device_reserved (GiB)": 34.19, + "memory/max_active (GiB)": 32.93, + "memory/max_allocated (GiB)": 32.93, + "ppl": 1.09754, + "step": 8, + "tokens/total": 120800, + "tokens/train_per_sec_per_gpu": 9.27, + "tokens/trainable": 2581 + } + ], + "logging_steps": 1, + "max_steps": 2048, + "num_input_tokens_seen": 0, + "num_train_epochs": 32, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8289979426483200.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/aft/coin/checkpoint-8/training_args.bin b/aft/coin/checkpoint-8/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a778af8c9efc3386896cecbfad59c1fffd3c766b --- /dev/null +++ b/aft/coin/checkpoint-8/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59deaf5eeeca5ea87ce1e8345e44acbe71a90824d99bc055c66361974780afd9 +size 8337