diff --git a/.gitattributes b/.gitattributes index 8e99a7578c023af7fa44533f9ad3e77b649d089d..47b23854c2b0936e0bfd735bec07f621385f78c1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -111,3 +111,11 @@ full_aft_midtrain4/coin4/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=l full_aft_midtrain4/coin4/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text full_aft_midtrain4/coin4/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text full_aft_midtrain4/coin4/checkpoint-8/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-128/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-16/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text +full_aft_midtrain4/balanced/checkpoint-8/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/full_aft_midtrain4/balanced/checkpoint-128/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-128/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/config.json b/full_aft_midtrain4/balanced/checkpoint-128/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-128/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-128/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..836b5e41cf48277cc576a69462bac1f57b1eb634 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a5e0af959288727d79345f566740842ad2f95dd4de24a5cf311bee9dbda9635 +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-128/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-128/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-128/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-128/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-128/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-128/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-128/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..238310a159c566a370ee3826e5319f667333704f --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/tokens_state.json @@ -0,0 +1 @@ +{"total": 2796262, "trainable": 58534} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-128/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-128/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f45fae2387e5687e151686b88806db45bc032afb --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/trainer_state.json @@ -0,0 +1,1826 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 128, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 12.4375, + "learning_rate": 5e-06, + "loss": 0.023131251335144043, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0234, + "step": 17, + "tokens/total": 369888, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.02522754669189453, + "memory/device_reserved (GiB)": 57.25, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02555, + "step": 18, + "tokens/total": 392455, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 8.4375, + "learning_rate": 5e-06, + "loss": 0.028311967849731445, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02872, + "step": 19, + "tokens/total": 413549, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 4.34375, + "learning_rate": 5e-06, + "loss": 0.012633323669433594, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01271, + "step": 20, + "tokens/total": 434128, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 3.34375, + "learning_rate": 5e-06, + "loss": 0.010331928730010986, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01039, + "step": 21, + "tokens/total": 456131, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 8.6875, + "learning_rate": 5e-06, + "loss": 0.011040687561035156, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0111, + "step": 22, + "tokens/total": 478707, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 17.75, + "learning_rate": 5e-06, + "loss": 0.030008912086486816, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03046, + "step": 23, + "tokens/total": 501425, + "tokens/train_per_sec_per_gpu": 4.9, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 6.6875, + "learning_rate": 5e-06, + "loss": 0.007420599460601807, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00745, + "step": 24, + "tokens/total": 522885, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 24.75, + "learning_rate": 5e-06, + "loss": 0.05265447497367859, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05407, + "step": 25, + "tokens/total": 544886, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 12.625, + "learning_rate": 5e-06, + "loss": 0.04731038212776184, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04845, + "step": 26, + "tokens/total": 564964, + "tokens/train_per_sec_per_gpu": 3.36, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 13.1875, + "learning_rate": 5e-06, + "loss": 0.014921486377716064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01503, + "step": 27, + "tokens/total": 586567, + "tokens/train_per_sec_per_gpu": 5.29, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 18.625, + "learning_rate": 5e-06, + "loss": 0.013006031513214111, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01309, + "step": 28, + "tokens/total": 608188, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 21.25, + "learning_rate": 5e-06, + "loss": 0.028101369738578796, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0285, + "step": 29, + "tokens/total": 630007, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 8.3125, + "learning_rate": 5e-06, + "loss": 0.01690027117729187, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01704, + "step": 30, + "tokens/total": 651656, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.015246853232383728, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01536, + "step": 31, + "tokens/total": 672938, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.017684906721115112, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01784, + "step": 32, + "tokens/total": 696525, + "tokens/train_per_sec_per_gpu": 5.03, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 15.5, + "learning_rate": 5e-06, + "loss": 0.02578166127204895, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02612, + "step": 33, + "tokens/total": 717452, + "tokens/train_per_sec_per_gpu": 2.67, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 10.5625, + "learning_rate": 5e-06, + "loss": 0.03840416669845581, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03915, + "step": 34, + "tokens/total": 738762, + "tokens/train_per_sec_per_gpu": 3.32, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 3.921875, + "learning_rate": 5e-06, + "loss": 0.017821550369262695, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01798, + "step": 35, + "tokens/total": 762239, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 5.71875, + "learning_rate": 5e-06, + "loss": 0.02945883572101593, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0299, + "step": 36, + "tokens/total": 784340, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 24.625, + "learning_rate": 5e-06, + "loss": 0.011434584856033325, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0115, + "step": 37, + "tokens/total": 806003, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.016611874103546143, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01675, + "step": 38, + "tokens/total": 828092, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 5.53125, + "learning_rate": 5e-06, + "loss": 0.012658625841140747, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 39, + "tokens/total": 849305, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 8.8125, + "learning_rate": 5e-06, + "loss": 0.02379515767097473, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02408, + "step": 40, + "tokens/total": 871803, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 2.015625, + "learning_rate": 5e-06, + "loss": 0.0050678253173828125, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00508, + "step": 41, + "tokens/total": 893895, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 5.875, + "learning_rate": 5e-06, + "loss": 0.016993582248687744, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01714, + "step": 42, + "tokens/total": 914803, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 4.1875, + "learning_rate": 5e-06, + "loss": 0.025514155626296997, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02584, + "step": 43, + "tokens/total": 938486, + "tokens/train_per_sec_per_gpu": 4.88, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 10.1875, + "learning_rate": 5e-06, + "loss": 0.013556092977523804, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01365, + "step": 44, + "tokens/total": 961149, + "tokens/train_per_sec_per_gpu": 4.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.012663766741752625, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 45, + "tokens/total": 982727, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 2.421875, + "learning_rate": 5e-06, + "loss": 0.00803878903388977, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00807, + "step": 46, + "tokens/total": 1003446, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 6.40625, + "learning_rate": 5e-06, + "loss": 0.027271926403045654, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02765, + "step": 47, + "tokens/total": 1024518, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 6.90625, + "learning_rate": 5e-06, + "loss": 0.019454196095466614, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01964, + "step": 48, + "tokens/total": 1046348, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.007123962044715881, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00715, + "step": 49, + "tokens/total": 1067448, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.016036301851272583, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01617, + "step": 50, + "tokens/total": 1091250, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.006014183163642883, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00603, + "step": 51, + "tokens/total": 1112130, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 8.5, + "learning_rate": 5e-06, + "loss": 0.01116333156824112, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01123, + "step": 52, + "tokens/total": 1135386, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 7.53125, + "learning_rate": 5e-06, + "loss": 0.012836582958698273, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01292, + "step": 53, + "tokens/total": 1156121, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.005587518215179443, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0056, + "step": 54, + "tokens/total": 1176702, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 78.5, + "learning_rate": 5e-06, + "loss": 0.05362102389335632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05508, + "step": 55, + "tokens/total": 1198839, + "tokens/train_per_sec_per_gpu": 3.13, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 2.796875, + "learning_rate": 5e-06, + "loss": 0.005013041198253632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00503, + "step": 56, + "tokens/total": 1219997, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.65234375, + "learning_rate": 5e-06, + "loss": 0.0010031796991825104, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.001, + "step": 57, + "tokens/total": 1241006, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 2.1875, + "learning_rate": 5e-06, + "loss": 0.0032418183982372284, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00325, + "step": 58, + "tokens/total": 1263530, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 15.3125, + "learning_rate": 5e-06, + "loss": 0.007594488561153412, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00762, + "step": 59, + "tokens/total": 1287978, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.009997941553592682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01005, + "step": 60, + "tokens/total": 1309463, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 2.34375, + "learning_rate": 5e-06, + "loss": 0.0018807370215654373, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00188, + "step": 61, + "tokens/total": 1331283, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 11.6875, + "learning_rate": 5e-06, + "loss": 0.03929818421602249, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04008, + "step": 62, + "tokens/total": 1352434, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.03207191079854965, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03259, + "step": 63, + "tokens/total": 1376383, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 27.125, + "learning_rate": 5e-06, + "loss": 0.03780709207057953, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03853, + "step": 64, + "tokens/total": 1399140, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 9.375, + "learning_rate": 5e-06, + "loss": 0.005781790241599083, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0058, + "step": 65, + "tokens/total": 1421624, + "tokens/train_per_sec_per_gpu": 3.27, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 5.78125, + "learning_rate": 5e-06, + "loss": 0.002019934356212616, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 66, + "tokens/total": 1442453, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 22.25, + "learning_rate": 5e-06, + "loss": 0.005651544779539108, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00567, + "step": 67, + "tokens/total": 1462799, + "tokens/train_per_sec_per_gpu": 5.22, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 14.8125, + "learning_rate": 5e-06, + "loss": 0.01753600873053074, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01769, + "step": 68, + "tokens/total": 1485719, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.011547420173883438, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01161, + "step": 69, + "tokens/total": 1509089, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 5.6875, + "learning_rate": 5e-06, + "loss": 0.010879918932914734, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01094, + "step": 70, + "tokens/total": 1530488, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 34.75, + "learning_rate": 5e-06, + "loss": 0.01058020070195198, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01064, + "step": 71, + "tokens/total": 1549610, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 7.40625, + "learning_rate": 5e-06, + "loss": 0.008541440591216087, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00858, + "step": 72, + "tokens/total": 1571082, + "tokens/train_per_sec_per_gpu": 3.96, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 4.375, + "learning_rate": 5e-06, + "loss": 0.011486759409308434, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01155, + "step": 73, + "tokens/total": 1591765, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 2.9375, + "learning_rate": 5e-06, + "loss": 0.0029051266610622406, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00291, + "step": 74, + "tokens/total": 1613822, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.007972124963998795, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.008, + "step": 75, + "tokens/total": 1634169, + "tokens/train_per_sec_per_gpu": 4.61, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 12.0625, + "learning_rate": 5e-06, + "loss": 0.016271991655230522, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01641, + "step": 76, + "tokens/total": 1654482, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 3.46875, + "learning_rate": 5e-06, + "loss": 0.006356913596391678, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00638, + "step": 77, + "tokens/total": 1675738, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 9.0625, + "learning_rate": 5e-06, + "loss": 0.026207175105810165, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02655, + "step": 78, + "tokens/total": 1697159, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 9.0, + "learning_rate": 5e-06, + "loss": 0.01054411381483078, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0106, + "step": 79, + "tokens/total": 1720420, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 6.4375, + "learning_rate": 5e-06, + "loss": 0.003844674676656723, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00385, + "step": 80, + "tokens/total": 1741815, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 13.5, + "learning_rate": 5e-06, + "loss": 0.010209254920482635, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01026, + "step": 81, + "tokens/total": 1762269, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.0027688145637512207, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00277, + "step": 82, + "tokens/total": 1785551, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.00830920785665512, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00834, + "step": 83, + "tokens/total": 1807278, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 6.46875, + "learning_rate": 5e-06, + "loss": 0.002562195062637329, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00257, + "step": 84, + "tokens/total": 1829716, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 10.4375, + "learning_rate": 5e-06, + "loss": 0.029578130692243576, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03002, + "step": 85, + "tokens/total": 1852531, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 11.75, + "learning_rate": 5e-06, + "loss": 0.024770356714725494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02508, + "step": 86, + "tokens/total": 1875946, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.003416106104850769, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00342, + "step": 87, + "tokens/total": 1897851, + "tokens/train_per_sec_per_gpu": 4.96, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 49.75, + "learning_rate": 5e-06, + "loss": 0.0025530681014060974, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00256, + "step": 88, + "tokens/total": 1918887, + "tokens/train_per_sec_per_gpu": 3.94, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.0036525018513202667, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00366, + "step": 89, + "tokens/total": 1941639, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 22.75, + "learning_rate": 5e-06, + "loss": 0.003507021814584732, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00351, + "step": 90, + "tokens/total": 1961837, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.443359375, + "learning_rate": 5e-06, + "loss": 0.0008387751877307892, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00084, + "step": 91, + "tokens/total": 1986102, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 2.75, + "learning_rate": 5e-06, + "loss": 0.0063427165150642395, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00636, + "step": 92, + "tokens/total": 2008498, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 8.375, + "learning_rate": 5e-06, + "loss": 0.0007749572396278381, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00078, + "step": 93, + "tokens/total": 2030279, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 4.90625, + "learning_rate": 5e-06, + "loss": 0.02601948380470276, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02636, + "step": 94, + "tokens/total": 2051914, + "tokens/train_per_sec_per_gpu": 5.33, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.015122350305318832, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01524, + "step": 95, + "tokens/total": 2071768, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 11.0625, + "learning_rate": 5e-06, + "loss": 0.01082584261894226, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01088, + "step": 96, + "tokens/total": 2094957, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 3.953125, + "learning_rate": 5e-06, + "loss": 0.008022107183933258, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00805, + "step": 97, + "tokens/total": 2117603, + "tokens/train_per_sec_per_gpu": 2.97, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 8.875, + "learning_rate": 5e-06, + "loss": 0.010948814451694489, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01101, + "step": 98, + "tokens/total": 2138416, + "tokens/train_per_sec_per_gpu": 4.0, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.01736413687467575, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01752, + "step": 99, + "tokens/total": 2160046, + "tokens/train_per_sec_per_gpu": 5.63, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 1.828125, + "learning_rate": 5e-06, + "loss": 0.002016715705394745, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 100, + "tokens/total": 2181923, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 1.25, + "learning_rate": 5e-06, + "loss": 0.0019299909472465515, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00193, + "step": 101, + "tokens/total": 2203558, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 3.859375, + "learning_rate": 5e-06, + "loss": 0.004272259771823883, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00428, + "step": 102, + "tokens/total": 2226903, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 8.5625, + "learning_rate": 5e-06, + "loss": 0.005463987588882446, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00548, + "step": 103, + "tokens/total": 2248454, + "tokens/train_per_sec_per_gpu": 4.56, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 28.25, + "learning_rate": 5e-06, + "loss": 0.029229726642370224, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02966, + "step": 104, + "tokens/total": 2271600, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.002427205443382263, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00243, + "step": 105, + "tokens/total": 2295278, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 3.734375, + "learning_rate": 5e-06, + "loss": 0.0015581585466861725, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00156, + "step": 106, + "tokens/total": 2317645, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 17.125, + "learning_rate": 5e-06, + "loss": 0.023405246436595917, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02368, + "step": 107, + "tokens/total": 2339279, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 2.53125, + "learning_rate": 5e-06, + "loss": 0.0015513133257627487, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00155, + "step": 108, + "tokens/total": 2360540, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 12.875, + "learning_rate": 5e-06, + "loss": 0.031832728534936905, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03234, + "step": 109, + "tokens/total": 2381374, + "tokens/train_per_sec_per_gpu": 4.73, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.002377200871706009, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00238, + "step": 110, + "tokens/total": 2403924, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 1.1875, + "learning_rate": 5e-06, + "loss": 0.0006611049175262451, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00066, + "step": 111, + "tokens/total": 2424591, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 1.359375, + "learning_rate": 5e-06, + "loss": 0.0006331540644168854, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00063, + "step": 112, + "tokens/total": 2446829, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.1259765625, + "learning_rate": 5e-06, + "loss": 0.0002308972179889679, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 113, + "tokens/total": 2467554, + "tokens/train_per_sec_per_gpu": 5.3, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.53515625, + "learning_rate": 5e-06, + "loss": 0.00044569000601768494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00045, + "step": 114, + "tokens/total": 2490093, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.640625, + "learning_rate": 5e-06, + "loss": 0.000523759052157402, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00052, + "step": 115, + "tokens/total": 2511764, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 10.3125, + "learning_rate": 5e-06, + "loss": 0.010446945205330849, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0105, + "step": 116, + "tokens/total": 2533541, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.0046858396381139755, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0047, + "step": 117, + "tokens/total": 2556046, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 4.40625, + "learning_rate": 5e-06, + "loss": 0.006048301234841347, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00607, + "step": 118, + "tokens/total": 2577387, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 3.328125, + "learning_rate": 5e-06, + "loss": 0.005638599395751953, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00565, + "step": 119, + "tokens/total": 2599912, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 12.5625, + "learning_rate": 5e-06, + "loss": 0.004470301792025566, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00448, + "step": 120, + "tokens/total": 2622592, + "tokens/train_per_sec_per_gpu": 5.57, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.007355847395956516, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00738, + "step": 121, + "tokens/total": 2646024, + "tokens/train_per_sec_per_gpu": 5.73, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 2.125, + "learning_rate": 5e-06, + "loss": 0.0003274921327829361, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 122, + "tokens/total": 2666034, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 3.421875, + "learning_rate": 5e-06, + "loss": 0.0009368900209665298, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00094, + "step": 123, + "tokens/total": 2687583, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.52734375, + "learning_rate": 5e-06, + "loss": 0.0003253612667322159, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 124, + "tokens/total": 2709310, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 22.625, + "learning_rate": 5e-06, + "loss": 0.0033743418753147125, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00338, + "step": 125, + "tokens/total": 2731725, + "tokens/train_per_sec_per_gpu": 5.25, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 23.625, + "learning_rate": 5e-06, + "loss": 0.005839193239808083, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00586, + "step": 126, + "tokens/total": 2754818, + "tokens/train_per_sec_per_gpu": 5.01, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 1.953125, + "learning_rate": 5e-06, + "loss": 0.0013373112305998802, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00134, + "step": 127, + "tokens/total": 2775918, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 5.90625, + "learning_rate": 5e-06, + "loss": 0.002514144405722618, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00252, + "step": 128, + "tokens/total": 2796262, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 58534 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8750157626330317e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-128/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-128/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-128/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-16/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-16/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/config.json b/full_aft_midtrain4/balanced/checkpoint-16/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-16/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-16/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0137609c24aa0fe6b91ce9b5e61ed14c312580e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:002935554ab69eb9b7e6de5d0d95d0a99a92cd58095478eb7c0861022d0dc671 +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-16/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-16/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-16/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-16/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-16/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-16/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-16/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..99e1af15d833f50188ac75651e7c5f553ce04cc1 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/tokens_state.json @@ -0,0 +1 @@ +{"total": 347838, "trainable": 7258} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-16/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-16/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9de229cf0af3bf412594290f66dc328cf5aa9b54 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/trainer_state.json @@ -0,0 +1,258 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.0625, + "eval_steps": 500, + "global_step": 16, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.332405666100019e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-16/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-16/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-16/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-256/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/config.json b/full_aft_midtrain4/balanced/checkpoint-256/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-256/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-256/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2cd5480aa628f622da69d9be1b4389d2b6f090d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dda9059ae44de482f67e01b52924e31cb4c5bb5646c6a1152be142261507ece +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-256/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-256/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-256/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-256/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-256/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a980df20d57cdc5507473372b811cfb301e21a56 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/tokens_state.json @@ -0,0 +1 @@ +{"total": 5602336, "trainable": 117030} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-256/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6dc83319b0c53690335be3013df1a0aa54448ff3 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/trainer_state.json @@ -0,0 +1,3618 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 12.4375, + "learning_rate": 5e-06, + "loss": 0.023131251335144043, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0234, + "step": 17, + "tokens/total": 369888, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.02522754669189453, + "memory/device_reserved (GiB)": 57.25, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02555, + "step": 18, + "tokens/total": 392455, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 8.4375, + "learning_rate": 5e-06, + "loss": 0.028311967849731445, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02872, + "step": 19, + "tokens/total": 413549, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 4.34375, + "learning_rate": 5e-06, + "loss": 0.012633323669433594, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01271, + "step": 20, + "tokens/total": 434128, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 3.34375, + "learning_rate": 5e-06, + "loss": 0.010331928730010986, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01039, + "step": 21, + "tokens/total": 456131, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 8.6875, + "learning_rate": 5e-06, + "loss": 0.011040687561035156, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0111, + "step": 22, + "tokens/total": 478707, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 17.75, + "learning_rate": 5e-06, + "loss": 0.030008912086486816, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03046, + "step": 23, + "tokens/total": 501425, + "tokens/train_per_sec_per_gpu": 4.9, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 6.6875, + "learning_rate": 5e-06, + "loss": 0.007420599460601807, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00745, + "step": 24, + "tokens/total": 522885, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 24.75, + "learning_rate": 5e-06, + "loss": 0.05265447497367859, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05407, + "step": 25, + "tokens/total": 544886, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 12.625, + "learning_rate": 5e-06, + "loss": 0.04731038212776184, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04845, + "step": 26, + "tokens/total": 564964, + "tokens/train_per_sec_per_gpu": 3.36, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 13.1875, + "learning_rate": 5e-06, + "loss": 0.014921486377716064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01503, + "step": 27, + "tokens/total": 586567, + "tokens/train_per_sec_per_gpu": 5.29, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 18.625, + "learning_rate": 5e-06, + "loss": 0.013006031513214111, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01309, + "step": 28, + "tokens/total": 608188, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 21.25, + "learning_rate": 5e-06, + "loss": 0.028101369738578796, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0285, + "step": 29, + "tokens/total": 630007, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 8.3125, + "learning_rate": 5e-06, + "loss": 0.01690027117729187, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01704, + "step": 30, + "tokens/total": 651656, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.015246853232383728, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01536, + "step": 31, + "tokens/total": 672938, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.017684906721115112, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01784, + "step": 32, + "tokens/total": 696525, + "tokens/train_per_sec_per_gpu": 5.03, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 15.5, + "learning_rate": 5e-06, + "loss": 0.02578166127204895, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02612, + "step": 33, + "tokens/total": 717452, + "tokens/train_per_sec_per_gpu": 2.67, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 10.5625, + "learning_rate": 5e-06, + "loss": 0.03840416669845581, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03915, + "step": 34, + "tokens/total": 738762, + "tokens/train_per_sec_per_gpu": 3.32, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 3.921875, + "learning_rate": 5e-06, + "loss": 0.017821550369262695, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01798, + "step": 35, + "tokens/total": 762239, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 5.71875, + "learning_rate": 5e-06, + "loss": 0.02945883572101593, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0299, + "step": 36, + "tokens/total": 784340, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 24.625, + "learning_rate": 5e-06, + "loss": 0.011434584856033325, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0115, + "step": 37, + "tokens/total": 806003, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.016611874103546143, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01675, + "step": 38, + "tokens/total": 828092, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 5.53125, + "learning_rate": 5e-06, + "loss": 0.012658625841140747, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 39, + "tokens/total": 849305, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 8.8125, + "learning_rate": 5e-06, + "loss": 0.02379515767097473, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02408, + "step": 40, + "tokens/total": 871803, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 2.015625, + "learning_rate": 5e-06, + "loss": 0.0050678253173828125, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00508, + "step": 41, + "tokens/total": 893895, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 5.875, + "learning_rate": 5e-06, + "loss": 0.016993582248687744, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01714, + "step": 42, + "tokens/total": 914803, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 4.1875, + "learning_rate": 5e-06, + "loss": 0.025514155626296997, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02584, + "step": 43, + "tokens/total": 938486, + "tokens/train_per_sec_per_gpu": 4.88, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 10.1875, + "learning_rate": 5e-06, + "loss": 0.013556092977523804, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01365, + "step": 44, + "tokens/total": 961149, + "tokens/train_per_sec_per_gpu": 4.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.012663766741752625, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 45, + "tokens/total": 982727, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 2.421875, + "learning_rate": 5e-06, + "loss": 0.00803878903388977, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00807, + "step": 46, + "tokens/total": 1003446, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 6.40625, + "learning_rate": 5e-06, + "loss": 0.027271926403045654, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02765, + "step": 47, + "tokens/total": 1024518, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 6.90625, + "learning_rate": 5e-06, + "loss": 0.019454196095466614, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01964, + "step": 48, + "tokens/total": 1046348, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.007123962044715881, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00715, + "step": 49, + "tokens/total": 1067448, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.016036301851272583, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01617, + "step": 50, + "tokens/total": 1091250, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.006014183163642883, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00603, + "step": 51, + "tokens/total": 1112130, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 8.5, + "learning_rate": 5e-06, + "loss": 0.01116333156824112, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01123, + "step": 52, + "tokens/total": 1135386, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 7.53125, + "learning_rate": 5e-06, + "loss": 0.012836582958698273, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01292, + "step": 53, + "tokens/total": 1156121, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.005587518215179443, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0056, + "step": 54, + "tokens/total": 1176702, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 78.5, + "learning_rate": 5e-06, + "loss": 0.05362102389335632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05508, + "step": 55, + "tokens/total": 1198839, + "tokens/train_per_sec_per_gpu": 3.13, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 2.796875, + "learning_rate": 5e-06, + "loss": 0.005013041198253632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00503, + "step": 56, + "tokens/total": 1219997, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.65234375, + "learning_rate": 5e-06, + "loss": 0.0010031796991825104, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.001, + "step": 57, + "tokens/total": 1241006, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 2.1875, + "learning_rate": 5e-06, + "loss": 0.0032418183982372284, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00325, + "step": 58, + "tokens/total": 1263530, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 15.3125, + "learning_rate": 5e-06, + "loss": 0.007594488561153412, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00762, + "step": 59, + "tokens/total": 1287978, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.009997941553592682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01005, + "step": 60, + "tokens/total": 1309463, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 2.34375, + "learning_rate": 5e-06, + "loss": 0.0018807370215654373, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00188, + "step": 61, + "tokens/total": 1331283, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 11.6875, + "learning_rate": 5e-06, + "loss": 0.03929818421602249, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04008, + "step": 62, + "tokens/total": 1352434, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.03207191079854965, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03259, + "step": 63, + "tokens/total": 1376383, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 27.125, + "learning_rate": 5e-06, + "loss": 0.03780709207057953, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03853, + "step": 64, + "tokens/total": 1399140, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 9.375, + "learning_rate": 5e-06, + "loss": 0.005781790241599083, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0058, + "step": 65, + "tokens/total": 1421624, + "tokens/train_per_sec_per_gpu": 3.27, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 5.78125, + "learning_rate": 5e-06, + "loss": 0.002019934356212616, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 66, + "tokens/total": 1442453, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 22.25, + "learning_rate": 5e-06, + "loss": 0.005651544779539108, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00567, + "step": 67, + "tokens/total": 1462799, + "tokens/train_per_sec_per_gpu": 5.22, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 14.8125, + "learning_rate": 5e-06, + "loss": 0.01753600873053074, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01769, + "step": 68, + "tokens/total": 1485719, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.011547420173883438, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01161, + "step": 69, + "tokens/total": 1509089, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 5.6875, + "learning_rate": 5e-06, + "loss": 0.010879918932914734, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01094, + "step": 70, + "tokens/total": 1530488, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 34.75, + "learning_rate": 5e-06, + "loss": 0.01058020070195198, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01064, + "step": 71, + "tokens/total": 1549610, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 7.40625, + "learning_rate": 5e-06, + "loss": 0.008541440591216087, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00858, + "step": 72, + "tokens/total": 1571082, + "tokens/train_per_sec_per_gpu": 3.96, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 4.375, + "learning_rate": 5e-06, + "loss": 0.011486759409308434, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01155, + "step": 73, + "tokens/total": 1591765, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 2.9375, + "learning_rate": 5e-06, + "loss": 0.0029051266610622406, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00291, + "step": 74, + "tokens/total": 1613822, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.007972124963998795, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.008, + "step": 75, + "tokens/total": 1634169, + "tokens/train_per_sec_per_gpu": 4.61, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 12.0625, + "learning_rate": 5e-06, + "loss": 0.016271991655230522, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01641, + "step": 76, + "tokens/total": 1654482, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 3.46875, + "learning_rate": 5e-06, + "loss": 0.006356913596391678, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00638, + "step": 77, + "tokens/total": 1675738, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 9.0625, + "learning_rate": 5e-06, + "loss": 0.026207175105810165, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02655, + "step": 78, + "tokens/total": 1697159, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 9.0, + "learning_rate": 5e-06, + "loss": 0.01054411381483078, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0106, + "step": 79, + "tokens/total": 1720420, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 6.4375, + "learning_rate": 5e-06, + "loss": 0.003844674676656723, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00385, + "step": 80, + "tokens/total": 1741815, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 13.5, + "learning_rate": 5e-06, + "loss": 0.010209254920482635, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01026, + "step": 81, + "tokens/total": 1762269, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.0027688145637512207, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00277, + "step": 82, + "tokens/total": 1785551, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.00830920785665512, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00834, + "step": 83, + "tokens/total": 1807278, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 6.46875, + "learning_rate": 5e-06, + "loss": 0.002562195062637329, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00257, + "step": 84, + "tokens/total": 1829716, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 10.4375, + "learning_rate": 5e-06, + "loss": 0.029578130692243576, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03002, + "step": 85, + "tokens/total": 1852531, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 11.75, + "learning_rate": 5e-06, + "loss": 0.024770356714725494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02508, + "step": 86, + "tokens/total": 1875946, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.003416106104850769, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00342, + "step": 87, + "tokens/total": 1897851, + "tokens/train_per_sec_per_gpu": 4.96, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 49.75, + "learning_rate": 5e-06, + "loss": 0.0025530681014060974, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00256, + "step": 88, + "tokens/total": 1918887, + "tokens/train_per_sec_per_gpu": 3.94, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.0036525018513202667, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00366, + "step": 89, + "tokens/total": 1941639, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 22.75, + "learning_rate": 5e-06, + "loss": 0.003507021814584732, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00351, + "step": 90, + "tokens/total": 1961837, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.443359375, + "learning_rate": 5e-06, + "loss": 0.0008387751877307892, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00084, + "step": 91, + "tokens/total": 1986102, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 2.75, + "learning_rate": 5e-06, + "loss": 0.0063427165150642395, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00636, + "step": 92, + "tokens/total": 2008498, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 8.375, + "learning_rate": 5e-06, + "loss": 0.0007749572396278381, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00078, + "step": 93, + "tokens/total": 2030279, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 4.90625, + "learning_rate": 5e-06, + "loss": 0.02601948380470276, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02636, + "step": 94, + "tokens/total": 2051914, + "tokens/train_per_sec_per_gpu": 5.33, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.015122350305318832, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01524, + "step": 95, + "tokens/total": 2071768, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 11.0625, + "learning_rate": 5e-06, + "loss": 0.01082584261894226, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01088, + "step": 96, + "tokens/total": 2094957, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 3.953125, + "learning_rate": 5e-06, + "loss": 0.008022107183933258, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00805, + "step": 97, + "tokens/total": 2117603, + "tokens/train_per_sec_per_gpu": 2.97, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 8.875, + "learning_rate": 5e-06, + "loss": 0.010948814451694489, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01101, + "step": 98, + "tokens/total": 2138416, + "tokens/train_per_sec_per_gpu": 4.0, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.01736413687467575, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01752, + "step": 99, + "tokens/total": 2160046, + "tokens/train_per_sec_per_gpu": 5.63, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 1.828125, + "learning_rate": 5e-06, + "loss": 0.002016715705394745, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 100, + "tokens/total": 2181923, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 1.25, + "learning_rate": 5e-06, + "loss": 0.0019299909472465515, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00193, + "step": 101, + "tokens/total": 2203558, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 3.859375, + "learning_rate": 5e-06, + "loss": 0.004272259771823883, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00428, + "step": 102, + "tokens/total": 2226903, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 8.5625, + "learning_rate": 5e-06, + "loss": 0.005463987588882446, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00548, + "step": 103, + "tokens/total": 2248454, + "tokens/train_per_sec_per_gpu": 4.56, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 28.25, + "learning_rate": 5e-06, + "loss": 0.029229726642370224, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02966, + "step": 104, + "tokens/total": 2271600, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.002427205443382263, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00243, + "step": 105, + "tokens/total": 2295278, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 3.734375, + "learning_rate": 5e-06, + "loss": 0.0015581585466861725, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00156, + "step": 106, + "tokens/total": 2317645, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 17.125, + "learning_rate": 5e-06, + "loss": 0.023405246436595917, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02368, + "step": 107, + "tokens/total": 2339279, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 2.53125, + "learning_rate": 5e-06, + "loss": 0.0015513133257627487, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00155, + "step": 108, + "tokens/total": 2360540, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 12.875, + "learning_rate": 5e-06, + "loss": 0.031832728534936905, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03234, + "step": 109, + "tokens/total": 2381374, + "tokens/train_per_sec_per_gpu": 4.73, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.002377200871706009, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00238, + "step": 110, + "tokens/total": 2403924, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 1.1875, + "learning_rate": 5e-06, + "loss": 0.0006611049175262451, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00066, + "step": 111, + "tokens/total": 2424591, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 1.359375, + "learning_rate": 5e-06, + "loss": 0.0006331540644168854, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00063, + "step": 112, + "tokens/total": 2446829, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.1259765625, + "learning_rate": 5e-06, + "loss": 0.0002308972179889679, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 113, + "tokens/total": 2467554, + "tokens/train_per_sec_per_gpu": 5.3, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.53515625, + "learning_rate": 5e-06, + "loss": 0.00044569000601768494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00045, + "step": 114, + "tokens/total": 2490093, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.640625, + "learning_rate": 5e-06, + "loss": 0.000523759052157402, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00052, + "step": 115, + "tokens/total": 2511764, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 10.3125, + "learning_rate": 5e-06, + "loss": 0.010446945205330849, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0105, + "step": 116, + "tokens/total": 2533541, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.0046858396381139755, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0047, + "step": 117, + "tokens/total": 2556046, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 4.40625, + "learning_rate": 5e-06, + "loss": 0.006048301234841347, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00607, + "step": 118, + "tokens/total": 2577387, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 3.328125, + "learning_rate": 5e-06, + "loss": 0.005638599395751953, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00565, + "step": 119, + "tokens/total": 2599912, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 12.5625, + "learning_rate": 5e-06, + "loss": 0.004470301792025566, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00448, + "step": 120, + "tokens/total": 2622592, + "tokens/train_per_sec_per_gpu": 5.57, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.007355847395956516, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00738, + "step": 121, + "tokens/total": 2646024, + "tokens/train_per_sec_per_gpu": 5.73, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 2.125, + "learning_rate": 5e-06, + "loss": 0.0003274921327829361, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 122, + "tokens/total": 2666034, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 3.421875, + "learning_rate": 5e-06, + "loss": 0.0009368900209665298, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00094, + "step": 123, + "tokens/total": 2687583, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.52734375, + "learning_rate": 5e-06, + "loss": 0.0003253612667322159, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 124, + "tokens/total": 2709310, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 22.625, + "learning_rate": 5e-06, + "loss": 0.0033743418753147125, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00338, + "step": 125, + "tokens/total": 2731725, + "tokens/train_per_sec_per_gpu": 5.25, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 23.625, + "learning_rate": 5e-06, + "loss": 0.005839193239808083, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00586, + "step": 126, + "tokens/total": 2754818, + "tokens/train_per_sec_per_gpu": 5.01, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 1.953125, + "learning_rate": 5e-06, + "loss": 0.0013373112305998802, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00134, + "step": 127, + "tokens/total": 2775918, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 5.90625, + "learning_rate": 5e-06, + "loss": 0.002514144405722618, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00252, + "step": 128, + "tokens/total": 2796262, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 1.2421875, + "learning_rate": 5e-06, + "loss": 0.0007996642962098122, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0008, + "step": 129, + "tokens/total": 2818125, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 6.84375, + "learning_rate": 5e-06, + "loss": 0.021156951785087585, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02138, + "step": 130, + "tokens/total": 2839638, + "tokens/train_per_sec_per_gpu": 4.7, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.006790724582970142, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00681, + "step": 131, + "tokens/total": 2861336, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 1.234375, + "learning_rate": 5e-06, + "loss": 0.0005378285422921181, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00054, + "step": 132, + "tokens/total": 2882775, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.42578125, + "learning_rate": 5e-06, + "loss": 0.00018137134611606598, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 133, + "tokens/total": 2905449, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.005508012138307095, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00552, + "step": 134, + "tokens/total": 2928542, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 3.84375, + "learning_rate": 5e-06, + "loss": 0.0016853846609592438, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00169, + "step": 135, + "tokens/total": 2951837, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.0011476436629891396, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00115, + "step": 136, + "tokens/total": 2973930, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 7.8125, + "learning_rate": 5e-06, + "loss": 0.004481654614210129, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00449, + "step": 137, + "tokens/total": 2995983, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 1.8359375, + "learning_rate": 5e-06, + "loss": 0.0007853899151086807, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00079, + "step": 138, + "tokens/total": 3017096, + "tokens/train_per_sec_per_gpu": 3.97, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 1.0390625, + "learning_rate": 5e-06, + "loss": 0.0002537788823246956, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00025, + "step": 139, + "tokens/total": 3037835, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 4.78125, + "learning_rate": 5e-06, + "loss": 0.02066297084093094, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02088, + "step": 140, + "tokens/total": 3060081, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.03966196998953819, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04046, + "step": 141, + "tokens/total": 3082713, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.0634765625, + "learning_rate": 5e-06, + "loss": 6.163213402032852e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 142, + "tokens/total": 3105112, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 1.2890625, + "learning_rate": 5e-06, + "loss": 0.0006557134911417961, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00066, + "step": 143, + "tokens/total": 3129163, + "tokens/train_per_sec_per_gpu": 5.68, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.318359375, + "learning_rate": 5e-06, + "loss": 0.00021572411060333252, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00022, + "step": 144, + "tokens/total": 3150107, + "tokens/train_per_sec_per_gpu": 3.98, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 30.75, + "learning_rate": 5e-06, + "loss": 0.01874990016222, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01893, + "step": 145, + "tokens/total": 3171894, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 8.0, + "learning_rate": 5e-06, + "loss": 0.007870376110076904, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0079, + "step": 146, + "tokens/total": 3193837, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 7.40625, + "learning_rate": 5e-06, + "loss": 0.01963133178651333, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01983, + "step": 147, + "tokens/total": 3215571, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 2.109375, + "learning_rate": 5e-06, + "loss": 0.0015872623771429062, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00159, + "step": 148, + "tokens/total": 3238021, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 10.4375, + "learning_rate": 5e-06, + "loss": 0.019420845434069633, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01961, + "step": 149, + "tokens/total": 3261456, + "tokens/train_per_sec_per_gpu": 3.18, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 3.59375, + "learning_rate": 5e-06, + "loss": 0.004275336861610413, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00428, + "step": 150, + "tokens/total": 3282559, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.330078125, + "learning_rate": 5e-06, + "loss": 0.0001280084252357483, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 151, + "tokens/total": 3302583, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.017270097509026527, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01742, + "step": 152, + "tokens/total": 3324861, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 1.140625, + "learning_rate": 5e-06, + "loss": 0.000630280002951622, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00063, + "step": 153, + "tokens/total": 3347335, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.048095703125, + "learning_rate": 5e-06, + "loss": 9.464658796787262e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 154, + "tokens/total": 3367165, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 2.328125, + "learning_rate": 5e-06, + "loss": 0.0009479951113462448, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00095, + "step": 155, + "tokens/total": 3389223, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 6.84375, + "learning_rate": 5e-06, + "loss": 0.0120626762509346, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01214, + "step": 156, + "tokens/total": 3410995, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.0072944387793540955, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00732, + "step": 157, + "tokens/total": 3431085, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 20.0, + "learning_rate": 5e-06, + "loss": 0.010568259283900261, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01062, + "step": 158, + "tokens/total": 3452324, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 3.15625, + "learning_rate": 5e-06, + "loss": 0.013493385165929794, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01358, + "step": 159, + "tokens/total": 3475279, + "tokens/train_per_sec_per_gpu": 4.94, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 2.21875, + "learning_rate": 5e-06, + "loss": 0.001761317253112793, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00176, + "step": 160, + "tokens/total": 3495486, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.072265625, + "learning_rate": 5e-06, + "loss": 0.0001285932958126068, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 161, + "tokens/total": 3517247, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 10.0, + "learning_rate": 5e-06, + "loss": 0.025099704042077065, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02542, + "step": 162, + "tokens/total": 3539179, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.66796875, + "learning_rate": 5e-06, + "loss": 0.0003115125000476837, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00031, + "step": 163, + "tokens/total": 3561165, + "tokens/train_per_sec_per_gpu": 4.98, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.42578125, + "learning_rate": 5e-06, + "loss": 0.0005782544612884521, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00058, + "step": 164, + "tokens/total": 3583859, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.51171875, + "learning_rate": 5e-06, + "loss": 0.00039725005626678467, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0004, + "step": 165, + "tokens/total": 3605644, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 4.6875, + "learning_rate": 5e-06, + "loss": 0.017310841009020805, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01746, + "step": 166, + "tokens/total": 3627496, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.2421875, + "learning_rate": 5e-06, + "loss": 0.0008921846747398376, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00089, + "step": 167, + "tokens/total": 3649200, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.376953125, + "learning_rate": 5e-06, + "loss": 0.00019433721899986267, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 168, + "tokens/total": 3668745, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.1953125, + "learning_rate": 5e-06, + "loss": 0.0002997666597366333, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0003, + "step": 169, + "tokens/total": 3690878, + "tokens/train_per_sec_per_gpu": 3.73, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.90234375, + "learning_rate": 5e-06, + "loss": 0.00048425979912281036, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00048, + "step": 170, + "tokens/total": 3712607, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 3.140625, + "learning_rate": 5e-06, + "loss": 0.002305535599589348, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00231, + "step": 171, + "tokens/total": 3735085, + "tokens/train_per_sec_per_gpu": 4.92, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.004036698490381241, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00404, + "step": 172, + "tokens/total": 3757110, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 5.5, + "learning_rate": 5e-06, + "loss": 0.004116147756576538, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00412, + "step": 173, + "tokens/total": 3780277, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 6.25, + "learning_rate": 5e-06, + "loss": 0.0032110102474689484, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00322, + "step": 174, + "tokens/total": 3801468, + "tokens/train_per_sec_per_gpu": 3.21, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.27734375, + "learning_rate": 5e-06, + "loss": 0.00018050335347652435, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 175, + "tokens/total": 3822213, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 11.4375, + "learning_rate": 5e-06, + "loss": 0.006123043596744537, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00614, + "step": 176, + "tokens/total": 3843234, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 2.5625, + "learning_rate": 5e-06, + "loss": 0.001666739583015442, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00167, + "step": 177, + "tokens/total": 3864599, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.078125, + "learning_rate": 5e-06, + "loss": 0.0003452356904745102, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00035, + "step": 178, + "tokens/total": 3884641, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.003512820228934288, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00352, + "step": 179, + "tokens/total": 3907103, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 7.09375, + "learning_rate": 5e-06, + "loss": 0.001678401604294777, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00168, + "step": 180, + "tokens/total": 3929962, + "tokens/train_per_sec_per_gpu": 3.75, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 18.875, + "learning_rate": 5e-06, + "loss": 0.009014192968606949, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00905, + "step": 181, + "tokens/total": 3950802, + "tokens/train_per_sec_per_gpu": 5.35, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.002051945775747299, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00205, + "step": 182, + "tokens/total": 3972850, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 5.03125, + "learning_rate": 5e-06, + "loss": 0.0022588688880205154, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00226, + "step": 183, + "tokens/total": 3996768, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 4.53125, + "learning_rate": 5e-06, + "loss": 0.0019080489873886108, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00191, + "step": 184, + "tokens/total": 4019000, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 3.671875, + "learning_rate": 5e-06, + "loss": 0.009887008927762508, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00994, + "step": 185, + "tokens/total": 4041530, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.01524898037314415, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01537, + "step": 186, + "tokens/total": 4063890, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.0654296875, + "learning_rate": 5e-06, + "loss": 9.179767221212387e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 187, + "tokens/total": 4087587, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 41.5, + "learning_rate": 5e-06, + "loss": 0.01077337097376585, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01083, + "step": 188, + "tokens/total": 4110005, + "tokens/train_per_sec_per_gpu": 5.34, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 2.296875, + "learning_rate": 5e-06, + "loss": 0.0011876951903104782, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00119, + "step": 189, + "tokens/total": 4130674, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04931640625, + "learning_rate": 5e-06, + "loss": 0.00010097958147525787, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 190, + "tokens/total": 4152766, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 1.78125, + "learning_rate": 5e-06, + "loss": 0.0009062271565198898, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00091, + "step": 191, + "tokens/total": 4173665, + "tokens/train_per_sec_per_gpu": 3.74, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 3.375, + "learning_rate": 5e-06, + "loss": 0.001665695570409298, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00167, + "step": 192, + "tokens/total": 4196778, + "tokens/train_per_sec_per_gpu": 5.25, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 5.375, + "learning_rate": 5e-06, + "loss": 0.0017034746706485748, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0017, + "step": 193, + "tokens/total": 4219974, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.10107421875, + "learning_rate": 5e-06, + "loss": 9.08784568309784e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 194, + "tokens/total": 4242748, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.63671875, + "learning_rate": 5e-06, + "loss": 0.00018034875392913818, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 195, + "tokens/total": 4262182, + "tokens/train_per_sec_per_gpu": 2.54, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.224609375, + "learning_rate": 5e-06, + "loss": 0.00022980757057666779, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 196, + "tokens/total": 4283330, + "tokens/train_per_sec_per_gpu": 4.6, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 3.15625, + "learning_rate": 5e-06, + "loss": 0.005374821834266186, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00539, + "step": 197, + "tokens/total": 4303754, + "tokens/train_per_sec_per_gpu": 3.18, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.0087890625, + "learning_rate": 5e-06, + "loss": 2.5076791644096375e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 198, + "tokens/total": 4325154, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.008272591978311539, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00831, + "step": 199, + "tokens/total": 4348423, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 4.125, + "learning_rate": 5e-06, + "loss": 0.021617580205202103, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02185, + "step": 200, + "tokens/total": 4369587, + "tokens/train_per_sec_per_gpu": 4.53, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 2.65625, + "learning_rate": 5e-06, + "loss": 0.0021339869126677513, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00214, + "step": 201, + "tokens/total": 4394085, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.18359375, + "learning_rate": 5e-06, + "loss": 0.0002014385536313057, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0002, + "step": 202, + "tokens/total": 4415941, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 10.0625, + "learning_rate": 5e-06, + "loss": 0.00018890481442213058, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 203, + "tokens/total": 4437541, + "tokens/train_per_sec_per_gpu": 5.06, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 1.921875, + "learning_rate": 5e-06, + "loss": 0.0017177853733301163, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00172, + "step": 204, + "tokens/total": 4460113, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 4.5, + "learning_rate": 5e-06, + "loss": 0.007489228621125221, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00752, + "step": 205, + "tokens/total": 4481669, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 7.96875, + "learning_rate": 5e-06, + "loss": 0.004295440390706062, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0043, + "step": 206, + "tokens/total": 4503055, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.24609375, + "learning_rate": 5e-06, + "loss": 0.00011829566210508347, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00012, + "step": 207, + "tokens/total": 4523908, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 1.0234375, + "learning_rate": 5e-06, + "loss": 0.0009613446891307831, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00096, + "step": 208, + "tokens/total": 4546626, + "tokens/train_per_sec_per_gpu": 5.7, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 2.125, + "learning_rate": 5e-06, + "loss": 0.0015333164483308792, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00153, + "step": 209, + "tokens/total": 4569735, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.2197265625, + "learning_rate": 5e-06, + "loss": 0.00019307155162096024, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 210, + "tokens/total": 4593226, + "tokens/train_per_sec_per_gpu": 5.0, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 4.59375, + "learning_rate": 5e-06, + "loss": 0.0038752183318138123, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00388, + "step": 211, + "tokens/total": 4614408, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 2.0625, + "learning_rate": 5e-06, + "loss": 0.0008769528940320015, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00088, + "step": 212, + "tokens/total": 4637147, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1953125, + "learning_rate": 5e-06, + "loss": 0.00014297012239694595, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00014, + "step": 213, + "tokens/total": 4656471, + "tokens/train_per_sec_per_gpu": 4.02, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.1435546875, + "learning_rate": 5e-06, + "loss": 0.00020532868802547455, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00021, + "step": 214, + "tokens/total": 4677621, + "tokens/train_per_sec_per_gpu": 4.71, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 3.5, + "learning_rate": 5e-06, + "loss": 0.004757304675877094, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00477, + "step": 215, + "tokens/total": 4700413, + "tokens/train_per_sec_per_gpu": 5.18, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.81640625, + "learning_rate": 5e-06, + "loss": 0.001078957226127386, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00108, + "step": 216, + "tokens/total": 4723445, + "tokens/train_per_sec_per_gpu": 3.2, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 4.65625, + "learning_rate": 5e-06, + "loss": 0.003942351788282394, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00395, + "step": 217, + "tokens/total": 4746122, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.055419921875, + "learning_rate": 5e-06, + "loss": 6.982963532209396e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 218, + "tokens/total": 4767157, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 1.6171875, + "learning_rate": 5e-06, + "loss": 0.00041304435580968857, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00041, + "step": 219, + "tokens/total": 4788211, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 3.109375, + "learning_rate": 5e-06, + "loss": 0.005511936265975237, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00553, + "step": 220, + "tokens/total": 4809595, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.031494140625, + "learning_rate": 5e-06, + "loss": 4.849769175052643e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 221, + "tokens/total": 4832024, + "tokens/train_per_sec_per_gpu": 4.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 1.640625, + "learning_rate": 5e-06, + "loss": 0.0009457077831029892, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00095, + "step": 222, + "tokens/total": 4853619, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 14.75, + "learning_rate": 5e-06, + "loss": 0.01612759381532669, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01626, + "step": 223, + "tokens/total": 4875686, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.044189453125, + "learning_rate": 5e-06, + "loss": 7.42003321647644e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 224, + "tokens/total": 4899377, + "tokens/train_per_sec_per_gpu": 5.61, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.515625, + "learning_rate": 5e-06, + "loss": 0.00015232432633638382, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00015, + "step": 225, + "tokens/total": 4921393, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.1064453125, + "learning_rate": 5e-06, + "loss": 0.00012585031799972057, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 226, + "tokens/total": 4943059, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 2.625, + "learning_rate": 5e-06, + "loss": 0.0016232873313128948, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00162, + "step": 227, + "tokens/total": 4965044, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 3.21875, + "learning_rate": 5e-06, + "loss": 0.001859381329268217, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00186, + "step": 228, + "tokens/total": 4987462, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 1.703125, + "learning_rate": 5e-06, + "loss": 0.00067939143627882, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00068, + "step": 229, + "tokens/total": 5010372, + "tokens/train_per_sec_per_gpu": 4.77, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 3.953125, + "learning_rate": 5e-06, + "loss": 0.013505356386303902, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0136, + "step": 230, + "tokens/total": 5032268, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0869140625, + "learning_rate": 5e-06, + "loss": 0.00010866299271583557, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00011, + "step": 231, + "tokens/total": 5051908, + "tokens/train_per_sec_per_gpu": 4.74, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.041259765625, + "learning_rate": 5e-06, + "loss": 3.86582687497139e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 232, + "tokens/total": 5073936, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.6015625, + "learning_rate": 5e-06, + "loss": 0.0002518047112971544, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00025, + "step": 233, + "tokens/total": 5096040, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 2.6875, + "learning_rate": 5e-06, + "loss": 0.0037277531810104847, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00373, + "step": 234, + "tokens/total": 5117335, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.72265625, + "learning_rate": 5e-06, + "loss": 0.00024031987413764, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00024, + "step": 235, + "tokens/total": 5138137, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 1.46875, + "learning_rate": 5e-06, + "loss": 0.00027213688008487225, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00027, + "step": 236, + "tokens/total": 5158794, + "tokens/train_per_sec_per_gpu": 5.83, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 14.125, + "learning_rate": 5e-06, + "loss": 0.022761110216379166, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02302, + "step": 237, + "tokens/total": 5180655, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.56640625, + "learning_rate": 5e-06, + "loss": 0.00024310685694217682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00024, + "step": 238, + "tokens/total": 5201887, + "tokens/train_per_sec_per_gpu": 4.64, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 1.6171875, + "learning_rate": 5e-06, + "loss": 0.0007034488953649998, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0007, + "step": 239, + "tokens/total": 5226350, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 4.6875, + "learning_rate": 5e-06, + "loss": 0.002927191089838743, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00293, + "step": 240, + "tokens/total": 5248328, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 3.640625, + "learning_rate": 5e-06, + "loss": 0.020369913429021835, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02058, + "step": 241, + "tokens/total": 5270552, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 3.140625, + "learning_rate": 5e-06, + "loss": 0.0010988134890794754, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0011, + "step": 242, + "tokens/total": 5290753, + "tokens/train_per_sec_per_gpu": 4.01, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 8.0625, + "learning_rate": 5e-06, + "loss": 0.030109882354736328, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03057, + "step": 243, + "tokens/total": 5313560, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 2.09375, + "learning_rate": 5e-06, + "loss": 0.002228438388556242, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00223, + "step": 244, + "tokens/total": 5335173, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.234375, + "learning_rate": 5e-06, + "loss": 0.00022633187472820282, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 245, + "tokens/total": 5358351, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 9.3125, + "learning_rate": 5e-06, + "loss": 0.026196565479040146, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02654, + "step": 246, + "tokens/total": 5379872, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 10.875, + "learning_rate": 5e-06, + "loss": 0.007026814855635166, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00705, + "step": 247, + "tokens/total": 5402193, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 16.625, + "learning_rate": 5e-06, + "loss": 0.013452365063130856, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01354, + "step": 248, + "tokens/total": 5426682, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.0046659111976623535, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00468, + "step": 249, + "tokens/total": 5451008, + "tokens/train_per_sec_per_gpu": 5.12, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.0030201319605112076, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00302, + "step": 250, + "tokens/total": 5472428, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 5.28125, + "learning_rate": 5e-06, + "loss": 0.008449372835457325, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00849, + "step": 251, + "tokens/total": 5495671, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 1.59375, + "learning_rate": 5e-06, + "loss": 0.0009228922426700592, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00092, + "step": 252, + "tokens/total": 5516185, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 7.9375, + "learning_rate": 5e-06, + "loss": 0.006769545841962099, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00679, + "step": 253, + "tokens/total": 5537676, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.47265625, + "learning_rate": 5e-06, + "loss": 0.0003159884363412857, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00032, + "step": 254, + "tokens/total": 5559478, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.0947265625, + "learning_rate": 5e-06, + "loss": 0.0001565776765346527, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00016, + "step": 255, + "tokens/total": 5579771, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.0021958593279123306, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0022, + "step": 256, + "tokens/total": 5602336, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 117030 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.7566109024518144e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-256/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-32/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/config.json b/full_aft_midtrain4/balanced/checkpoint-32/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-32/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-32/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0d89fd3c80eb12e63eb1a4e8a713c430d334fcee --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c03dd00e78c43dc4052cc08a04dd8f58588503ee28dd0f7ba7476925c61d4d +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-32/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-32/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-32/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-32/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-32/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-32/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..80c90e00eb480889c6c4b5989d906f9cd199f44b --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/tokens_state.json @@ -0,0 +1 @@ +{"total": 696525, "trainable": 14565} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-32/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-32/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..791286f616b06f069335adb6ed1948a06622d428 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/trainer_state.json @@ -0,0 +1,482 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.125, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 12.4375, + "learning_rate": 5e-06, + "loss": 0.023131251335144043, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0234, + "step": 17, + "tokens/total": 369888, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.02522754669189453, + "memory/device_reserved (GiB)": 57.25, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02555, + "step": 18, + "tokens/total": 392455, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 8.4375, + "learning_rate": 5e-06, + "loss": 0.028311967849731445, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02872, + "step": 19, + "tokens/total": 413549, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 4.34375, + "learning_rate": 5e-06, + "loss": 0.012633323669433594, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01271, + "step": 20, + "tokens/total": 434128, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 3.34375, + "learning_rate": 5e-06, + "loss": 0.010331928730010986, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01039, + "step": 21, + "tokens/total": 456131, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 8.6875, + "learning_rate": 5e-06, + "loss": 0.011040687561035156, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0111, + "step": 22, + "tokens/total": 478707, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 17.75, + "learning_rate": 5e-06, + "loss": 0.030008912086486816, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03046, + "step": 23, + "tokens/total": 501425, + "tokens/train_per_sec_per_gpu": 4.9, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 6.6875, + "learning_rate": 5e-06, + "loss": 0.007420599460601807, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00745, + "step": 24, + "tokens/total": 522885, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 24.75, + "learning_rate": 5e-06, + "loss": 0.05265447497367859, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05407, + "step": 25, + "tokens/total": 544886, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 12.625, + "learning_rate": 5e-06, + "loss": 0.04731038212776184, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04845, + "step": 26, + "tokens/total": 564964, + "tokens/train_per_sec_per_gpu": 3.36, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 13.1875, + "learning_rate": 5e-06, + "loss": 0.014921486377716064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01503, + "step": 27, + "tokens/total": 586567, + "tokens/train_per_sec_per_gpu": 5.29, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 18.625, + "learning_rate": 5e-06, + "loss": 0.013006031513214111, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01309, + "step": 28, + "tokens/total": 608188, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 21.25, + "learning_rate": 5e-06, + "loss": 0.028101369738578796, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0285, + "step": 29, + "tokens/total": 630007, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 8.3125, + "learning_rate": 5e-06, + "loss": 0.01690027117729187, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01704, + "step": 30, + "tokens/total": 651656, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.015246853232383728, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01536, + "step": 31, + "tokens/total": 672938, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.017684906721115112, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01784, + "step": 32, + "tokens/total": 696525, + "tokens/train_per_sec_per_gpu": 5.03, + "tokens/trainable": 14565 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.670504244242022e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-32/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-32/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-4/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/config.json b/full_aft_midtrain4/balanced/checkpoint-4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-4/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ba46e0f7133f1a67de2bb7b10e958a5642ebff93 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d4a0f1216763f701d3156eab5d12dd3f2ee3a75d28d01e2e80373cca8e39625 +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-4/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-4/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-4/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-4/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-4/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d9fab88fe596ec195997397f4daa64bbebe8789c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/tokens_state.json @@ -0,0 +1 @@ +{"total": 88800, "trainable": 1850} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-4/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-4/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dd8fd18b8a232f588a68bf8c7748d8d1da69ef47 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/trainer_state.json @@ -0,0 +1,90 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.015625, + "eval_steps": 500, + "global_step": 4, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5954427768799232.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-4/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-4/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-4/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-512/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-512/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/config.json b/full_aft_midtrain4/balanced/checkpoint-512/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-512/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-512/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4921ac48dacb9b414798f083ba81ae34f9f675c1 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bef7ed59a90a992d5b104c7b996d27c4e45aededcdbc806b184ea7d01dbc6a1 +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-512/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-512/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-512/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-512/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-512/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-512/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-512/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..88ea704619814e0bb91856069489888e35f54576 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/tokens_state.json @@ -0,0 +1 @@ +{"total": 11204672, "trainable": 234060} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-512/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-512/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4b1302a16a563372d6f387117271e5f6bb072c00 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/trainer_state.json @@ -0,0 +1,7202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 12.4375, + "learning_rate": 5e-06, + "loss": 0.023131251335144043, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0234, + "step": 17, + "tokens/total": 369888, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.02522754669189453, + "memory/device_reserved (GiB)": 57.25, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02555, + "step": 18, + "tokens/total": 392455, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 8.4375, + "learning_rate": 5e-06, + "loss": 0.028311967849731445, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02872, + "step": 19, + "tokens/total": 413549, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 4.34375, + "learning_rate": 5e-06, + "loss": 0.012633323669433594, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01271, + "step": 20, + "tokens/total": 434128, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 3.34375, + "learning_rate": 5e-06, + "loss": 0.010331928730010986, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01039, + "step": 21, + "tokens/total": 456131, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 8.6875, + "learning_rate": 5e-06, + "loss": 0.011040687561035156, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0111, + "step": 22, + "tokens/total": 478707, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 17.75, + "learning_rate": 5e-06, + "loss": 0.030008912086486816, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03046, + "step": 23, + "tokens/total": 501425, + "tokens/train_per_sec_per_gpu": 4.9, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 6.6875, + "learning_rate": 5e-06, + "loss": 0.007420599460601807, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00745, + "step": 24, + "tokens/total": 522885, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 24.75, + "learning_rate": 5e-06, + "loss": 0.05265447497367859, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05407, + "step": 25, + "tokens/total": 544886, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 12.625, + "learning_rate": 5e-06, + "loss": 0.04731038212776184, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04845, + "step": 26, + "tokens/total": 564964, + "tokens/train_per_sec_per_gpu": 3.36, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 13.1875, + "learning_rate": 5e-06, + "loss": 0.014921486377716064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01503, + "step": 27, + "tokens/total": 586567, + "tokens/train_per_sec_per_gpu": 5.29, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 18.625, + "learning_rate": 5e-06, + "loss": 0.013006031513214111, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01309, + "step": 28, + "tokens/total": 608188, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 21.25, + "learning_rate": 5e-06, + "loss": 0.028101369738578796, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0285, + "step": 29, + "tokens/total": 630007, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 8.3125, + "learning_rate": 5e-06, + "loss": 0.01690027117729187, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01704, + "step": 30, + "tokens/total": 651656, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.015246853232383728, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01536, + "step": 31, + "tokens/total": 672938, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.017684906721115112, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01784, + "step": 32, + "tokens/total": 696525, + "tokens/train_per_sec_per_gpu": 5.03, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 15.5, + "learning_rate": 5e-06, + "loss": 0.02578166127204895, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02612, + "step": 33, + "tokens/total": 717452, + "tokens/train_per_sec_per_gpu": 2.67, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 10.5625, + "learning_rate": 5e-06, + "loss": 0.03840416669845581, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03915, + "step": 34, + "tokens/total": 738762, + "tokens/train_per_sec_per_gpu": 3.32, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 3.921875, + "learning_rate": 5e-06, + "loss": 0.017821550369262695, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01798, + "step": 35, + "tokens/total": 762239, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 5.71875, + "learning_rate": 5e-06, + "loss": 0.02945883572101593, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0299, + "step": 36, + "tokens/total": 784340, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 24.625, + "learning_rate": 5e-06, + "loss": 0.011434584856033325, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0115, + "step": 37, + "tokens/total": 806003, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.016611874103546143, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01675, + "step": 38, + "tokens/total": 828092, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 5.53125, + "learning_rate": 5e-06, + "loss": 0.012658625841140747, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 39, + "tokens/total": 849305, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 8.8125, + "learning_rate": 5e-06, + "loss": 0.02379515767097473, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02408, + "step": 40, + "tokens/total": 871803, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 2.015625, + "learning_rate": 5e-06, + "loss": 0.0050678253173828125, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00508, + "step": 41, + "tokens/total": 893895, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 5.875, + "learning_rate": 5e-06, + "loss": 0.016993582248687744, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01714, + "step": 42, + "tokens/total": 914803, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 4.1875, + "learning_rate": 5e-06, + "loss": 0.025514155626296997, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02584, + "step": 43, + "tokens/total": 938486, + "tokens/train_per_sec_per_gpu": 4.88, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 10.1875, + "learning_rate": 5e-06, + "loss": 0.013556092977523804, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01365, + "step": 44, + "tokens/total": 961149, + "tokens/train_per_sec_per_gpu": 4.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.012663766741752625, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 45, + "tokens/total": 982727, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 2.421875, + "learning_rate": 5e-06, + "loss": 0.00803878903388977, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00807, + "step": 46, + "tokens/total": 1003446, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 6.40625, + "learning_rate": 5e-06, + "loss": 0.027271926403045654, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02765, + "step": 47, + "tokens/total": 1024518, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 6.90625, + "learning_rate": 5e-06, + "loss": 0.019454196095466614, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01964, + "step": 48, + "tokens/total": 1046348, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.007123962044715881, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00715, + "step": 49, + "tokens/total": 1067448, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.016036301851272583, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01617, + "step": 50, + "tokens/total": 1091250, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.006014183163642883, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00603, + "step": 51, + "tokens/total": 1112130, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 8.5, + "learning_rate": 5e-06, + "loss": 0.01116333156824112, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01123, + "step": 52, + "tokens/total": 1135386, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 7.53125, + "learning_rate": 5e-06, + "loss": 0.012836582958698273, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01292, + "step": 53, + "tokens/total": 1156121, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.005587518215179443, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0056, + "step": 54, + "tokens/total": 1176702, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 78.5, + "learning_rate": 5e-06, + "loss": 0.05362102389335632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05508, + "step": 55, + "tokens/total": 1198839, + "tokens/train_per_sec_per_gpu": 3.13, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 2.796875, + "learning_rate": 5e-06, + "loss": 0.005013041198253632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00503, + "step": 56, + "tokens/total": 1219997, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.65234375, + "learning_rate": 5e-06, + "loss": 0.0010031796991825104, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.001, + "step": 57, + "tokens/total": 1241006, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 2.1875, + "learning_rate": 5e-06, + "loss": 0.0032418183982372284, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00325, + "step": 58, + "tokens/total": 1263530, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 15.3125, + "learning_rate": 5e-06, + "loss": 0.007594488561153412, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00762, + "step": 59, + "tokens/total": 1287978, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.009997941553592682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01005, + "step": 60, + "tokens/total": 1309463, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 2.34375, + "learning_rate": 5e-06, + "loss": 0.0018807370215654373, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00188, + "step": 61, + "tokens/total": 1331283, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 11.6875, + "learning_rate": 5e-06, + "loss": 0.03929818421602249, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04008, + "step": 62, + "tokens/total": 1352434, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.03207191079854965, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03259, + "step": 63, + "tokens/total": 1376383, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 27.125, + "learning_rate": 5e-06, + "loss": 0.03780709207057953, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03853, + "step": 64, + "tokens/total": 1399140, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 9.375, + "learning_rate": 5e-06, + "loss": 0.005781790241599083, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0058, + "step": 65, + "tokens/total": 1421624, + "tokens/train_per_sec_per_gpu": 3.27, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 5.78125, + "learning_rate": 5e-06, + "loss": 0.002019934356212616, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 66, + "tokens/total": 1442453, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 22.25, + "learning_rate": 5e-06, + "loss": 0.005651544779539108, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00567, + "step": 67, + "tokens/total": 1462799, + "tokens/train_per_sec_per_gpu": 5.22, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 14.8125, + "learning_rate": 5e-06, + "loss": 0.01753600873053074, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01769, + "step": 68, + "tokens/total": 1485719, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.011547420173883438, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01161, + "step": 69, + "tokens/total": 1509089, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 5.6875, + "learning_rate": 5e-06, + "loss": 0.010879918932914734, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01094, + "step": 70, + "tokens/total": 1530488, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 34.75, + "learning_rate": 5e-06, + "loss": 0.01058020070195198, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01064, + "step": 71, + "tokens/total": 1549610, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 7.40625, + "learning_rate": 5e-06, + "loss": 0.008541440591216087, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00858, + "step": 72, + "tokens/total": 1571082, + "tokens/train_per_sec_per_gpu": 3.96, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 4.375, + "learning_rate": 5e-06, + "loss": 0.011486759409308434, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01155, + "step": 73, + "tokens/total": 1591765, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 2.9375, + "learning_rate": 5e-06, + "loss": 0.0029051266610622406, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00291, + "step": 74, + "tokens/total": 1613822, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.007972124963998795, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.008, + "step": 75, + "tokens/total": 1634169, + "tokens/train_per_sec_per_gpu": 4.61, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 12.0625, + "learning_rate": 5e-06, + "loss": 0.016271991655230522, + "memory/device_reserved (GiB)": 58.2, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01641, + "step": 76, + "tokens/total": 1654482, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 3.46875, + "learning_rate": 5e-06, + "loss": 0.006356913596391678, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00638, + "step": 77, + "tokens/total": 1675738, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 9.0625, + "learning_rate": 5e-06, + "loss": 0.026207175105810165, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02655, + "step": 78, + "tokens/total": 1697159, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 9.0, + "learning_rate": 5e-06, + "loss": 0.01054411381483078, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0106, + "step": 79, + "tokens/total": 1720420, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 6.4375, + "learning_rate": 5e-06, + "loss": 0.003844674676656723, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00385, + "step": 80, + "tokens/total": 1741815, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 13.5, + "learning_rate": 5e-06, + "loss": 0.010209254920482635, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01026, + "step": 81, + "tokens/total": 1762269, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.0027688145637512207, + "memory/device_reserved (GiB)": 59.92, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00277, + "step": 82, + "tokens/total": 1785551, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.00830920785665512, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00834, + "step": 83, + "tokens/total": 1807278, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 6.46875, + "learning_rate": 5e-06, + "loss": 0.002562195062637329, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00257, + "step": 84, + "tokens/total": 1829716, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 10.4375, + "learning_rate": 5e-06, + "loss": 0.029578130692243576, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03002, + "step": 85, + "tokens/total": 1852531, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 11.75, + "learning_rate": 5e-06, + "loss": 0.024770356714725494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02508, + "step": 86, + "tokens/total": 1875946, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.003416106104850769, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00342, + "step": 87, + "tokens/total": 1897851, + "tokens/train_per_sec_per_gpu": 4.96, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 49.75, + "learning_rate": 5e-06, + "loss": 0.0025530681014060974, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00256, + "step": 88, + "tokens/total": 1918887, + "tokens/train_per_sec_per_gpu": 3.94, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.0036525018513202667, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00366, + "step": 89, + "tokens/total": 1941639, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 22.75, + "learning_rate": 5e-06, + "loss": 0.003507021814584732, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00351, + "step": 90, + "tokens/total": 1961837, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.443359375, + "learning_rate": 5e-06, + "loss": 0.0008387751877307892, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00084, + "step": 91, + "tokens/total": 1986102, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 2.75, + "learning_rate": 5e-06, + "loss": 0.0063427165150642395, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00636, + "step": 92, + "tokens/total": 2008498, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 8.375, + "learning_rate": 5e-06, + "loss": 0.0007749572396278381, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00078, + "step": 93, + "tokens/total": 2030279, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 4.90625, + "learning_rate": 5e-06, + "loss": 0.02601948380470276, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02636, + "step": 94, + "tokens/total": 2051914, + "tokens/train_per_sec_per_gpu": 5.33, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.015122350305318832, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01524, + "step": 95, + "tokens/total": 2071768, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 11.0625, + "learning_rate": 5e-06, + "loss": 0.01082584261894226, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01088, + "step": 96, + "tokens/total": 2094957, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 3.953125, + "learning_rate": 5e-06, + "loss": 0.008022107183933258, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00805, + "step": 97, + "tokens/total": 2117603, + "tokens/train_per_sec_per_gpu": 2.97, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 8.875, + "learning_rate": 5e-06, + "loss": 0.010948814451694489, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01101, + "step": 98, + "tokens/total": 2138416, + "tokens/train_per_sec_per_gpu": 4.0, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.01736413687467575, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01752, + "step": 99, + "tokens/total": 2160046, + "tokens/train_per_sec_per_gpu": 5.63, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 1.828125, + "learning_rate": 5e-06, + "loss": 0.002016715705394745, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00202, + "step": 100, + "tokens/total": 2181923, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 1.25, + "learning_rate": 5e-06, + "loss": 0.0019299909472465515, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00193, + "step": 101, + "tokens/total": 2203558, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 3.859375, + "learning_rate": 5e-06, + "loss": 0.004272259771823883, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00428, + "step": 102, + "tokens/total": 2226903, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 8.5625, + "learning_rate": 5e-06, + "loss": 0.005463987588882446, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00548, + "step": 103, + "tokens/total": 2248454, + "tokens/train_per_sec_per_gpu": 4.56, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 28.25, + "learning_rate": 5e-06, + "loss": 0.029229726642370224, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02966, + "step": 104, + "tokens/total": 2271600, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 3.8125, + "learning_rate": 5e-06, + "loss": 0.002427205443382263, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00243, + "step": 105, + "tokens/total": 2295278, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 3.734375, + "learning_rate": 5e-06, + "loss": 0.0015581585466861725, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00156, + "step": 106, + "tokens/total": 2317645, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 17.125, + "learning_rate": 5e-06, + "loss": 0.023405246436595917, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02368, + "step": 107, + "tokens/total": 2339279, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 2.53125, + "learning_rate": 5e-06, + "loss": 0.0015513133257627487, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00155, + "step": 108, + "tokens/total": 2360540, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 12.875, + "learning_rate": 5e-06, + "loss": 0.031832728534936905, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03234, + "step": 109, + "tokens/total": 2381374, + "tokens/train_per_sec_per_gpu": 4.73, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.002377200871706009, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00238, + "step": 110, + "tokens/total": 2403924, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 1.1875, + "learning_rate": 5e-06, + "loss": 0.0006611049175262451, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00066, + "step": 111, + "tokens/total": 2424591, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 1.359375, + "learning_rate": 5e-06, + "loss": 0.0006331540644168854, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00063, + "step": 112, + "tokens/total": 2446829, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.1259765625, + "learning_rate": 5e-06, + "loss": 0.0002308972179889679, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 113, + "tokens/total": 2467554, + "tokens/train_per_sec_per_gpu": 5.3, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.53515625, + "learning_rate": 5e-06, + "loss": 0.00044569000601768494, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00045, + "step": 114, + "tokens/total": 2490093, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.640625, + "learning_rate": 5e-06, + "loss": 0.000523759052157402, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00052, + "step": 115, + "tokens/total": 2511764, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 10.3125, + "learning_rate": 5e-06, + "loss": 0.010446945205330849, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0105, + "step": 116, + "tokens/total": 2533541, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.0046858396381139755, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0047, + "step": 117, + "tokens/total": 2556046, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 4.40625, + "learning_rate": 5e-06, + "loss": 0.006048301234841347, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00607, + "step": 118, + "tokens/total": 2577387, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 3.328125, + "learning_rate": 5e-06, + "loss": 0.005638599395751953, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00565, + "step": 119, + "tokens/total": 2599912, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 12.5625, + "learning_rate": 5e-06, + "loss": 0.004470301792025566, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00448, + "step": 120, + "tokens/total": 2622592, + "tokens/train_per_sec_per_gpu": 5.57, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 10.25, + "learning_rate": 5e-06, + "loss": 0.007355847395956516, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00738, + "step": 121, + "tokens/total": 2646024, + "tokens/train_per_sec_per_gpu": 5.73, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 2.125, + "learning_rate": 5e-06, + "loss": 0.0003274921327829361, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 122, + "tokens/total": 2666034, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 3.421875, + "learning_rate": 5e-06, + "loss": 0.0009368900209665298, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00094, + "step": 123, + "tokens/total": 2687583, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.52734375, + "learning_rate": 5e-06, + "loss": 0.0003253612667322159, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 124, + "tokens/total": 2709310, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 22.625, + "learning_rate": 5e-06, + "loss": 0.0033743418753147125, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00338, + "step": 125, + "tokens/total": 2731725, + "tokens/train_per_sec_per_gpu": 5.25, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 23.625, + "learning_rate": 5e-06, + "loss": 0.005839193239808083, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00586, + "step": 126, + "tokens/total": 2754818, + "tokens/train_per_sec_per_gpu": 5.01, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 1.953125, + "learning_rate": 5e-06, + "loss": 0.0013373112305998802, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00134, + "step": 127, + "tokens/total": 2775918, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 5.90625, + "learning_rate": 5e-06, + "loss": 0.002514144405722618, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00252, + "step": 128, + "tokens/total": 2796262, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 1.2421875, + "learning_rate": 5e-06, + "loss": 0.0007996642962098122, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0008, + "step": 129, + "tokens/total": 2818125, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 6.84375, + "learning_rate": 5e-06, + "loss": 0.021156951785087585, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02138, + "step": 130, + "tokens/total": 2839638, + "tokens/train_per_sec_per_gpu": 4.7, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.006790724582970142, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00681, + "step": 131, + "tokens/total": 2861336, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 1.234375, + "learning_rate": 5e-06, + "loss": 0.0005378285422921181, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00054, + "step": 132, + "tokens/total": 2882775, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.42578125, + "learning_rate": 5e-06, + "loss": 0.00018137134611606598, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 133, + "tokens/total": 2905449, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.005508012138307095, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00552, + "step": 134, + "tokens/total": 2928542, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 3.84375, + "learning_rate": 5e-06, + "loss": 0.0016853846609592438, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00169, + "step": 135, + "tokens/total": 2951837, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.0011476436629891396, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00115, + "step": 136, + "tokens/total": 2973930, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 7.8125, + "learning_rate": 5e-06, + "loss": 0.004481654614210129, + "memory/device_reserved (GiB)": 58.98, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00449, + "step": 137, + "tokens/total": 2995983, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 1.8359375, + "learning_rate": 5e-06, + "loss": 0.0007853899151086807, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00079, + "step": 138, + "tokens/total": 3017096, + "tokens/train_per_sec_per_gpu": 3.97, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 1.0390625, + "learning_rate": 5e-06, + "loss": 0.0002537788823246956, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00025, + "step": 139, + "tokens/total": 3037835, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 4.78125, + "learning_rate": 5e-06, + "loss": 0.02066297084093094, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02088, + "step": 140, + "tokens/total": 3060081, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.03966196998953819, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04046, + "step": 141, + "tokens/total": 3082713, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.0634765625, + "learning_rate": 5e-06, + "loss": 6.163213402032852e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 142, + "tokens/total": 3105112, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 1.2890625, + "learning_rate": 5e-06, + "loss": 0.0006557134911417961, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00066, + "step": 143, + "tokens/total": 3129163, + "tokens/train_per_sec_per_gpu": 5.68, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.318359375, + "learning_rate": 5e-06, + "loss": 0.00021572411060333252, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00022, + "step": 144, + "tokens/total": 3150107, + "tokens/train_per_sec_per_gpu": 3.98, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 30.75, + "learning_rate": 5e-06, + "loss": 0.01874990016222, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01893, + "step": 145, + "tokens/total": 3171894, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 8.0, + "learning_rate": 5e-06, + "loss": 0.007870376110076904, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0079, + "step": 146, + "tokens/total": 3193837, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 7.40625, + "learning_rate": 5e-06, + "loss": 0.01963133178651333, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01983, + "step": 147, + "tokens/total": 3215571, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 2.109375, + "learning_rate": 5e-06, + "loss": 0.0015872623771429062, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00159, + "step": 148, + "tokens/total": 3238021, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 10.4375, + "learning_rate": 5e-06, + "loss": 0.019420845434069633, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01961, + "step": 149, + "tokens/total": 3261456, + "tokens/train_per_sec_per_gpu": 3.18, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 3.59375, + "learning_rate": 5e-06, + "loss": 0.004275336861610413, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00428, + "step": 150, + "tokens/total": 3282559, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.330078125, + "learning_rate": 5e-06, + "loss": 0.0001280084252357483, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 151, + "tokens/total": 3302583, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 6.375, + "learning_rate": 5e-06, + "loss": 0.017270097509026527, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01742, + "step": 152, + "tokens/total": 3324861, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 1.140625, + "learning_rate": 5e-06, + "loss": 0.000630280002951622, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00063, + "step": 153, + "tokens/total": 3347335, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.048095703125, + "learning_rate": 5e-06, + "loss": 9.464658796787262e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 154, + "tokens/total": 3367165, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 2.328125, + "learning_rate": 5e-06, + "loss": 0.0009479951113462448, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00095, + "step": 155, + "tokens/total": 3389223, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 6.84375, + "learning_rate": 5e-06, + "loss": 0.0120626762509346, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01214, + "step": 156, + "tokens/total": 3410995, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.0072944387793540955, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00732, + "step": 157, + "tokens/total": 3431085, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 20.0, + "learning_rate": 5e-06, + "loss": 0.010568259283900261, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01062, + "step": 158, + "tokens/total": 3452324, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 3.15625, + "learning_rate": 5e-06, + "loss": 0.013493385165929794, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01358, + "step": 159, + "tokens/total": 3475279, + "tokens/train_per_sec_per_gpu": 4.94, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 2.21875, + "learning_rate": 5e-06, + "loss": 0.001761317253112793, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00176, + "step": 160, + "tokens/total": 3495486, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.072265625, + "learning_rate": 5e-06, + "loss": 0.0001285932958126068, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 161, + "tokens/total": 3517247, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 10.0, + "learning_rate": 5e-06, + "loss": 0.025099704042077065, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02542, + "step": 162, + "tokens/total": 3539179, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.66796875, + "learning_rate": 5e-06, + "loss": 0.0003115125000476837, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00031, + "step": 163, + "tokens/total": 3561165, + "tokens/train_per_sec_per_gpu": 4.98, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.42578125, + "learning_rate": 5e-06, + "loss": 0.0005782544612884521, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00058, + "step": 164, + "tokens/total": 3583859, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.51171875, + "learning_rate": 5e-06, + "loss": 0.00039725005626678467, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0004, + "step": 165, + "tokens/total": 3605644, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 4.6875, + "learning_rate": 5e-06, + "loss": 0.017310841009020805, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01746, + "step": 166, + "tokens/total": 3627496, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.2421875, + "learning_rate": 5e-06, + "loss": 0.0008921846747398376, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00089, + "step": 167, + "tokens/total": 3649200, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.376953125, + "learning_rate": 5e-06, + "loss": 0.00019433721899986267, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 168, + "tokens/total": 3668745, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.1953125, + "learning_rate": 5e-06, + "loss": 0.0002997666597366333, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0003, + "step": 169, + "tokens/total": 3690878, + "tokens/train_per_sec_per_gpu": 3.73, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.90234375, + "learning_rate": 5e-06, + "loss": 0.00048425979912281036, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00048, + "step": 170, + "tokens/total": 3712607, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 3.140625, + "learning_rate": 5e-06, + "loss": 0.002305535599589348, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00231, + "step": 171, + "tokens/total": 3735085, + "tokens/train_per_sec_per_gpu": 4.92, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.004036698490381241, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00404, + "step": 172, + "tokens/total": 3757110, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 5.5, + "learning_rate": 5e-06, + "loss": 0.004116147756576538, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00412, + "step": 173, + "tokens/total": 3780277, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 6.25, + "learning_rate": 5e-06, + "loss": 0.0032110102474689484, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00322, + "step": 174, + "tokens/total": 3801468, + "tokens/train_per_sec_per_gpu": 3.21, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.27734375, + "learning_rate": 5e-06, + "loss": 0.00018050335347652435, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 175, + "tokens/total": 3822213, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 11.4375, + "learning_rate": 5e-06, + "loss": 0.006123043596744537, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00614, + "step": 176, + "tokens/total": 3843234, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 2.5625, + "learning_rate": 5e-06, + "loss": 0.001666739583015442, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00167, + "step": 177, + "tokens/total": 3864599, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.078125, + "learning_rate": 5e-06, + "loss": 0.0003452356904745102, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00035, + "step": 178, + "tokens/total": 3884641, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 4.71875, + "learning_rate": 5e-06, + "loss": 0.003512820228934288, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00352, + "step": 179, + "tokens/total": 3907103, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 7.09375, + "learning_rate": 5e-06, + "loss": 0.001678401604294777, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00168, + "step": 180, + "tokens/total": 3929962, + "tokens/train_per_sec_per_gpu": 3.75, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 18.875, + "learning_rate": 5e-06, + "loss": 0.009014192968606949, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00905, + "step": 181, + "tokens/total": 3950802, + "tokens/train_per_sec_per_gpu": 5.35, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 7.0, + "learning_rate": 5e-06, + "loss": 0.002051945775747299, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00205, + "step": 182, + "tokens/total": 3972850, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 5.03125, + "learning_rate": 5e-06, + "loss": 0.0022588688880205154, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00226, + "step": 183, + "tokens/total": 3996768, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 4.53125, + "learning_rate": 5e-06, + "loss": 0.0019080489873886108, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00191, + "step": 184, + "tokens/total": 4019000, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 3.671875, + "learning_rate": 5e-06, + "loss": 0.009887008927762508, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00994, + "step": 185, + "tokens/total": 4041530, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.01524898037314415, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01537, + "step": 186, + "tokens/total": 4063890, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.0654296875, + "learning_rate": 5e-06, + "loss": 9.179767221212387e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 187, + "tokens/total": 4087587, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 41.5, + "learning_rate": 5e-06, + "loss": 0.01077337097376585, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01083, + "step": 188, + "tokens/total": 4110005, + "tokens/train_per_sec_per_gpu": 5.34, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 2.296875, + "learning_rate": 5e-06, + "loss": 0.0011876951903104782, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00119, + "step": 189, + "tokens/total": 4130674, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04931640625, + "learning_rate": 5e-06, + "loss": 0.00010097958147525787, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 190, + "tokens/total": 4152766, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 1.78125, + "learning_rate": 5e-06, + "loss": 0.0009062271565198898, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00091, + "step": 191, + "tokens/total": 4173665, + "tokens/train_per_sec_per_gpu": 3.74, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 3.375, + "learning_rate": 5e-06, + "loss": 0.001665695570409298, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00167, + "step": 192, + "tokens/total": 4196778, + "tokens/train_per_sec_per_gpu": 5.25, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 5.375, + "learning_rate": 5e-06, + "loss": 0.0017034746706485748, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0017, + "step": 193, + "tokens/total": 4219974, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.10107421875, + "learning_rate": 5e-06, + "loss": 9.08784568309784e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 194, + "tokens/total": 4242748, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.63671875, + "learning_rate": 5e-06, + "loss": 0.00018034875392913818, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 195, + "tokens/total": 4262182, + "tokens/train_per_sec_per_gpu": 2.54, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.224609375, + "learning_rate": 5e-06, + "loss": 0.00022980757057666779, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 196, + "tokens/total": 4283330, + "tokens/train_per_sec_per_gpu": 4.6, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 3.15625, + "learning_rate": 5e-06, + "loss": 0.005374821834266186, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00539, + "step": 197, + "tokens/total": 4303754, + "tokens/train_per_sec_per_gpu": 3.18, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.0087890625, + "learning_rate": 5e-06, + "loss": 2.5076791644096375e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 198, + "tokens/total": 4325154, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.008272591978311539, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00831, + "step": 199, + "tokens/total": 4348423, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 4.125, + "learning_rate": 5e-06, + "loss": 0.021617580205202103, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02185, + "step": 200, + "tokens/total": 4369587, + "tokens/train_per_sec_per_gpu": 4.53, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 2.65625, + "learning_rate": 5e-06, + "loss": 0.0021339869126677513, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00214, + "step": 201, + "tokens/total": 4394085, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.18359375, + "learning_rate": 5e-06, + "loss": 0.0002014385536313057, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0002, + "step": 202, + "tokens/total": 4415941, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 10.0625, + "learning_rate": 5e-06, + "loss": 0.00018890481442213058, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 203, + "tokens/total": 4437541, + "tokens/train_per_sec_per_gpu": 5.06, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 1.921875, + "learning_rate": 5e-06, + "loss": 0.0017177853733301163, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00172, + "step": 204, + "tokens/total": 4460113, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 4.5, + "learning_rate": 5e-06, + "loss": 0.007489228621125221, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00752, + "step": 205, + "tokens/total": 4481669, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 7.96875, + "learning_rate": 5e-06, + "loss": 0.004295440390706062, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0043, + "step": 206, + "tokens/total": 4503055, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.24609375, + "learning_rate": 5e-06, + "loss": 0.00011829566210508347, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00012, + "step": 207, + "tokens/total": 4523908, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 1.0234375, + "learning_rate": 5e-06, + "loss": 0.0009613446891307831, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00096, + "step": 208, + "tokens/total": 4546626, + "tokens/train_per_sec_per_gpu": 5.7, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 2.125, + "learning_rate": 5e-06, + "loss": 0.0015333164483308792, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00153, + "step": 209, + "tokens/total": 4569735, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.2197265625, + "learning_rate": 5e-06, + "loss": 0.00019307155162096024, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 210, + "tokens/total": 4593226, + "tokens/train_per_sec_per_gpu": 5.0, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 4.59375, + "learning_rate": 5e-06, + "loss": 0.0038752183318138123, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00388, + "step": 211, + "tokens/total": 4614408, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 2.0625, + "learning_rate": 5e-06, + "loss": 0.0008769528940320015, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00088, + "step": 212, + "tokens/total": 4637147, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1953125, + "learning_rate": 5e-06, + "loss": 0.00014297012239694595, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00014, + "step": 213, + "tokens/total": 4656471, + "tokens/train_per_sec_per_gpu": 4.02, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.1435546875, + "learning_rate": 5e-06, + "loss": 0.00020532868802547455, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00021, + "step": 214, + "tokens/total": 4677621, + "tokens/train_per_sec_per_gpu": 4.71, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 3.5, + "learning_rate": 5e-06, + "loss": 0.004757304675877094, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00477, + "step": 215, + "tokens/total": 4700413, + "tokens/train_per_sec_per_gpu": 5.18, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.81640625, + "learning_rate": 5e-06, + "loss": 0.001078957226127386, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00108, + "step": 216, + "tokens/total": 4723445, + "tokens/train_per_sec_per_gpu": 3.2, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 4.65625, + "learning_rate": 5e-06, + "loss": 0.003942351788282394, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00395, + "step": 217, + "tokens/total": 4746122, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.055419921875, + "learning_rate": 5e-06, + "loss": 6.982963532209396e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 218, + "tokens/total": 4767157, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 1.6171875, + "learning_rate": 5e-06, + "loss": 0.00041304435580968857, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00041, + "step": 219, + "tokens/total": 4788211, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 3.109375, + "learning_rate": 5e-06, + "loss": 0.005511936265975237, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00553, + "step": 220, + "tokens/total": 4809595, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.031494140625, + "learning_rate": 5e-06, + "loss": 4.849769175052643e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 221, + "tokens/total": 4832024, + "tokens/train_per_sec_per_gpu": 4.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 1.640625, + "learning_rate": 5e-06, + "loss": 0.0009457077831029892, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00095, + "step": 222, + "tokens/total": 4853619, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 14.75, + "learning_rate": 5e-06, + "loss": 0.01612759381532669, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01626, + "step": 223, + "tokens/total": 4875686, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.044189453125, + "learning_rate": 5e-06, + "loss": 7.42003321647644e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 224, + "tokens/total": 4899377, + "tokens/train_per_sec_per_gpu": 5.61, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.515625, + "learning_rate": 5e-06, + "loss": 0.00015232432633638382, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00015, + "step": 225, + "tokens/total": 4921393, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.1064453125, + "learning_rate": 5e-06, + "loss": 0.00012585031799972057, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00013, + "step": 226, + "tokens/total": 4943059, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 2.625, + "learning_rate": 5e-06, + "loss": 0.0016232873313128948, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00162, + "step": 227, + "tokens/total": 4965044, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 3.21875, + "learning_rate": 5e-06, + "loss": 0.001859381329268217, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00186, + "step": 228, + "tokens/total": 4987462, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 1.703125, + "learning_rate": 5e-06, + "loss": 0.00067939143627882, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00068, + "step": 229, + "tokens/total": 5010372, + "tokens/train_per_sec_per_gpu": 4.77, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 3.953125, + "learning_rate": 5e-06, + "loss": 0.013505356386303902, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0136, + "step": 230, + "tokens/total": 5032268, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0869140625, + "learning_rate": 5e-06, + "loss": 0.00010866299271583557, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00011, + "step": 231, + "tokens/total": 5051908, + "tokens/train_per_sec_per_gpu": 4.74, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.041259765625, + "learning_rate": 5e-06, + "loss": 3.86582687497139e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 232, + "tokens/total": 5073936, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.6015625, + "learning_rate": 5e-06, + "loss": 0.0002518047112971544, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00025, + "step": 233, + "tokens/total": 5096040, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 2.6875, + "learning_rate": 5e-06, + "loss": 0.0037277531810104847, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00373, + "step": 234, + "tokens/total": 5117335, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.72265625, + "learning_rate": 5e-06, + "loss": 0.00024031987413764, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00024, + "step": 235, + "tokens/total": 5138137, + "tokens/train_per_sec_per_gpu": 3.91, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 1.46875, + "learning_rate": 5e-06, + "loss": 0.00027213688008487225, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00027, + "step": 236, + "tokens/total": 5158794, + "tokens/train_per_sec_per_gpu": 5.83, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 14.125, + "learning_rate": 5e-06, + "loss": 0.022761110216379166, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02302, + "step": 237, + "tokens/total": 5180655, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.56640625, + "learning_rate": 5e-06, + "loss": 0.00024310685694217682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00024, + "step": 238, + "tokens/total": 5201887, + "tokens/train_per_sec_per_gpu": 4.64, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 1.6171875, + "learning_rate": 5e-06, + "loss": 0.0007034488953649998, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0007, + "step": 239, + "tokens/total": 5226350, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 4.6875, + "learning_rate": 5e-06, + "loss": 0.002927191089838743, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00293, + "step": 240, + "tokens/total": 5248328, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 3.640625, + "learning_rate": 5e-06, + "loss": 0.020369913429021835, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02058, + "step": 241, + "tokens/total": 5270552, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 3.140625, + "learning_rate": 5e-06, + "loss": 0.0010988134890794754, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0011, + "step": 242, + "tokens/total": 5290753, + "tokens/train_per_sec_per_gpu": 4.01, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 8.0625, + "learning_rate": 5e-06, + "loss": 0.030109882354736328, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03057, + "step": 243, + "tokens/total": 5313560, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 2.09375, + "learning_rate": 5e-06, + "loss": 0.002228438388556242, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00223, + "step": 244, + "tokens/total": 5335173, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.234375, + "learning_rate": 5e-06, + "loss": 0.00022633187472820282, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 245, + "tokens/total": 5358351, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 9.3125, + "learning_rate": 5e-06, + "loss": 0.026196565479040146, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02654, + "step": 246, + "tokens/total": 5379872, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 10.875, + "learning_rate": 5e-06, + "loss": 0.007026814855635166, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00705, + "step": 247, + "tokens/total": 5402193, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 16.625, + "learning_rate": 5e-06, + "loss": 0.013452365063130856, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01354, + "step": 248, + "tokens/total": 5426682, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.0046659111976623535, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00468, + "step": 249, + "tokens/total": 5451008, + "tokens/train_per_sec_per_gpu": 5.12, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 7.5, + "learning_rate": 5e-06, + "loss": 0.0030201319605112076, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00302, + "step": 250, + "tokens/total": 5472428, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 5.28125, + "learning_rate": 5e-06, + "loss": 0.008449372835457325, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00849, + "step": 251, + "tokens/total": 5495671, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 1.59375, + "learning_rate": 5e-06, + "loss": 0.0009228922426700592, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00092, + "step": 252, + "tokens/total": 5516185, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 7.9375, + "learning_rate": 5e-06, + "loss": 0.006769545841962099, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00679, + "step": 253, + "tokens/total": 5537676, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.47265625, + "learning_rate": 5e-06, + "loss": 0.0003159884363412857, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00032, + "step": 254, + "tokens/total": 5559478, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.0947265625, + "learning_rate": 5e-06, + "loss": 0.0001565776765346527, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00016, + "step": 255, + "tokens/total": 5579771, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.0021958593279123306, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0022, + "step": 256, + "tokens/total": 5602336, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 3.1875, + "learning_rate": 5e-06, + "loss": 0.0015037236735224724, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0015, + "step": 257, + "tokens/total": 5626517, + "tokens/train_per_sec_per_gpu": 3.22, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 8.375, + "learning_rate": 5e-06, + "loss": 0.0028473520651459694, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00285, + "step": 258, + "tokens/total": 5649936, + "tokens/train_per_sec_per_gpu": 5.79, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 6.96875, + "learning_rate": 5e-06, + "loss": 0.0008803699165582657, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00088, + "step": 259, + "tokens/total": 5673917, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 0.83984375, + "learning_rate": 5e-06, + "loss": 0.0004903534427285194, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00049, + "step": 260, + "tokens/total": 5696410, + "tokens/train_per_sec_per_gpu": 4.26, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.462890625, + "learning_rate": 5e-06, + "loss": 0.000334043987095356, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00033, + "step": 261, + "tokens/total": 5718418, + "tokens/train_per_sec_per_gpu": 4.87, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.470703125, + "learning_rate": 5e-06, + "loss": 0.00037403404712677, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00037, + "step": 262, + "tokens/total": 5739430, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.068359375, + "learning_rate": 5e-06, + "loss": 9.411107748746872e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 263, + "tokens/total": 5758799, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 1.3125, + "learning_rate": 5e-06, + "loss": 0.0009417850524187088, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00094, + "step": 264, + "tokens/total": 5781129, + "tokens/train_per_sec_per_gpu": 3.28, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.0228271484375, + "learning_rate": 5e-06, + "loss": 4.118354991078377e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 265, + "tokens/total": 5802053, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.21484375, + "learning_rate": 5e-06, + "loss": 0.00023051584139466286, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 266, + "tokens/total": 5824523, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.375, + "learning_rate": 5e-06, + "loss": 0.00026505161076784134, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00027, + "step": 267, + "tokens/total": 5847293, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.013427734375, + "learning_rate": 5e-06, + "loss": 2.683699131011963e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 268, + "tokens/total": 5868224, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.0308837890625, + "learning_rate": 5e-06, + "loss": 5.9276819229125977e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 269, + "tokens/total": 5892576, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.01202392578125, + "learning_rate": 5e-06, + "loss": 2.9031652957201004e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 270, + "tokens/total": 5912598, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.279296875, + "learning_rate": 5e-06, + "loss": 0.0001440085470676422, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00014, + "step": 271, + "tokens/total": 5933856, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 6.75, + "learning_rate": 5e-06, + "loss": 0.005184470675885677, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0052, + "step": 272, + "tokens/total": 5955480, + "tokens/train_per_sec_per_gpu": 3.18, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.0947265625, + "learning_rate": 5e-06, + "loss": 9.596953168511391e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 273, + "tokens/total": 5978105, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 5.71875, + "learning_rate": 5e-06, + "loss": 0.0035652650985866785, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00357, + "step": 274, + "tokens/total": 6000531, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.019775390625, + "learning_rate": 5e-06, + "loss": 3.5872217267751694e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 275, + "tokens/total": 6024491, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 5.21875, + "learning_rate": 5e-06, + "loss": 0.005823442712426186, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00584, + "step": 276, + "tokens/total": 6047476, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 4.5625, + "learning_rate": 5e-06, + "loss": 0.0027098290156573057, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00271, + "step": 277, + "tokens/total": 6070343, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 8.375, + "learning_rate": 5e-06, + "loss": 0.008747010491788387, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00879, + "step": 278, + "tokens/total": 6090268, + "tokens/train_per_sec_per_gpu": 3.3, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.01397705078125, + "learning_rate": 5e-06, + "loss": 2.4213222786784172e-05, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 279, + "tokens/total": 6110715, + "tokens/train_per_sec_per_gpu": 5.24, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 4.0625, + "learning_rate": 5e-06, + "loss": 0.0013620443642139435, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00136, + "step": 280, + "tokens/total": 6131715, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 7.84375, + "learning_rate": 5e-06, + "loss": 0.007394424639642239, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00742, + "step": 281, + "tokens/total": 6153850, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 8.125, + "learning_rate": 5e-06, + "loss": 0.0050263721495866776, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00504, + "step": 282, + "tokens/total": 6175713, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 4.0, + "learning_rate": 5e-06, + "loss": 0.00087537313811481, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00088, + "step": 283, + "tokens/total": 6197566, + "tokens/train_per_sec_per_gpu": 5.01, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 3.421875, + "learning_rate": 5e-06, + "loss": 0.0031914650462567806, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0032, + "step": 284, + "tokens/total": 6218750, + "tokens/train_per_sec_per_gpu": 3.26, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.78125, + "learning_rate": 5e-06, + "loss": 0.0001507173292338848, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00015, + "step": 285, + "tokens/total": 6242400, + "tokens/train_per_sec_per_gpu": 4.97, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.0108642578125, + "learning_rate": 5e-06, + "loss": 2.374919131398201e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 286, + "tokens/total": 6265861, + "tokens/train_per_sec_per_gpu": 5.3, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.0791015625, + "learning_rate": 5e-06, + "loss": 7.562455721199512e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00008, + "step": 287, + "tokens/total": 6287541, + "tokens/train_per_sec_per_gpu": 4.62, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 21.75, + "learning_rate": 5e-06, + "loss": 0.04237286373972893, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04328, + "step": 288, + "tokens/total": 6307670, + "tokens/train_per_sec_per_gpu": 2.92, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.134765625, + "learning_rate": 5e-06, + "loss": 8.283811621367931e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00008, + "step": 289, + "tokens/total": 6329663, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.12353515625, + "learning_rate": 5e-06, + "loss": 5.346490070223808e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 290, + "tokens/total": 6351176, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.0211181640625, + "learning_rate": 5e-06, + "loss": 2.338876947760582e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 291, + "tokens/total": 6373670, + "tokens/train_per_sec_per_gpu": 3.21, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.0927734375, + "learning_rate": 5e-06, + "loss": 6.431248039007187e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 292, + "tokens/total": 6396405, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 3.15625, + "learning_rate": 5e-06, + "loss": 0.01033029519021511, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01038, + "step": 293, + "tokens/total": 6419979, + "tokens/train_per_sec_per_gpu": 5.0, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.01068115234375, + "learning_rate": 5e-06, + "loss": 1.8890248611569405e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 294, + "tokens/total": 6440802, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.13671875, + "learning_rate": 5e-06, + "loss": 6.03795051574707e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 295, + "tokens/total": 6461666, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 4.28125, + "learning_rate": 5e-06, + "loss": 0.004920740146189928, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00493, + "step": 296, + "tokens/total": 6482350, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 2.484375, + "learning_rate": 5e-06, + "loss": 0.005074569024145603, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00509, + "step": 297, + "tokens/total": 6501623, + "tokens/train_per_sec_per_gpu": 3.33, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 3.859375, + "learning_rate": 5e-06, + "loss": 0.0012149140238761902, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00122, + "step": 298, + "tokens/total": 6522074, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10595703125, + "learning_rate": 5e-06, + "loss": 6.340676918625832e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 299, + "tokens/total": 6544849, + "tokens/train_per_sec_per_gpu": 4.56, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0712890625, + "learning_rate": 5e-06, + "loss": 5.502719432115555e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 300, + "tokens/total": 6567278, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.3828125, + "learning_rate": 5e-06, + "loss": 0.00033586565405130386, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00034, + "step": 301, + "tokens/total": 6590168, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.6484375, + "learning_rate": 5e-06, + "loss": 0.0004671141505241394, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00047, + "step": 302, + "tokens/total": 6610760, + "tokens/train_per_sec_per_gpu": 3.26, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 4.8125, + "learning_rate": 5e-06, + "loss": 0.002447525504976511, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00245, + "step": 303, + "tokens/total": 6631816, + "tokens/train_per_sec_per_gpu": 3.94, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 5.875, + "learning_rate": 5e-06, + "loss": 0.014834374189376831, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01494, + "step": 304, + "tokens/total": 6652814, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.05517578125, + "learning_rate": 5e-06, + "loss": 8.606398478150368e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 305, + "tokens/total": 6674775, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.03564453125, + "learning_rate": 5e-06, + "loss": 5.543651059269905e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 306, + "tokens/total": 6695375, + "tokens/train_per_sec_per_gpu": 3.27, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.1005859375, + "learning_rate": 5e-06, + "loss": 9.982660412788391e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 307, + "tokens/total": 6716615, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0771484375, + "learning_rate": 5e-06, + "loss": 0.00011081807315349579, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00011, + "step": 308, + "tokens/total": 6740622, + "tokens/train_per_sec_per_gpu": 5.53, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.029541015625, + "learning_rate": 5e-06, + "loss": 5.179690197110176e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 309, + "tokens/total": 6762456, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.01953125, + "learning_rate": 5e-06, + "loss": 3.4774187952280045e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 310, + "tokens/total": 6783854, + "tokens/train_per_sec_per_gpu": 5.35, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.75, + "learning_rate": 5e-06, + "loss": 0.0004914090968668461, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00049, + "step": 311, + "tokens/total": 6804901, + "tokens/train_per_sec_per_gpu": 5.26, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.068359375, + "learning_rate": 5e-06, + "loss": 5.279621109366417e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 312, + "tokens/total": 6826598, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.003168296068906784, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00317, + "step": 313, + "tokens/total": 6849398, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.2080078125, + "learning_rate": 5e-06, + "loss": 9.494973346590996e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 314, + "tokens/total": 6870883, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.025146484375, + "learning_rate": 5e-06, + "loss": 4.329485818743706e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 315, + "tokens/total": 6893054, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.04931640625, + "learning_rate": 5e-06, + "loss": 4.947930574417114e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 316, + "tokens/total": 6915072, + "tokens/train_per_sec_per_gpu": 3.19, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.77734375, + "learning_rate": 5e-06, + "loss": 0.0003609228879213333, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00036, + "step": 317, + "tokens/total": 6937700, + "tokens/train_per_sec_per_gpu": 5.68, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.01171875, + "learning_rate": 5e-06, + "loss": 2.496456727385521e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 318, + "tokens/total": 6959215, + "tokens/train_per_sec_per_gpu": 3.21, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.037353515625, + "learning_rate": 5e-06, + "loss": 4.137866199016571e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 319, + "tokens/total": 6981968, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 1.0078125, + "learning_rate": 5e-06, + "loss": 0.0009954627603292465, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.001, + "step": 320, + "tokens/total": 7006582, + "tokens/train_per_sec_per_gpu": 4.18, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0240478515625, + "learning_rate": 5e-06, + "loss": 2.5064684450626373e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 321, + "tokens/total": 7028902, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.07861328125, + "learning_rate": 5e-06, + "loss": 5.47654926776886e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 322, + "tokens/total": 7050458, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.1787109375, + "learning_rate": 5e-06, + "loss": 9.893765673041344e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 323, + "tokens/total": 7072064, + "tokens/train_per_sec_per_gpu": 4.87, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 1.0546875, + "learning_rate": 5e-06, + "loss": 0.0006006578914821148, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0006, + "step": 324, + "tokens/total": 7092630, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0625, + "learning_rate": 5e-06, + "loss": 3.9601000025868416e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 325, + "tokens/total": 7113811, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 5.0, + "learning_rate": 5e-06, + "loss": 0.024376913905143738, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02468, + "step": 326, + "tokens/total": 7136195, + "tokens/train_per_sec_per_gpu": 4.27, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.048828125, + "learning_rate": 5e-06, + "loss": 4.375400021672249e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 327, + "tokens/total": 7157320, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 2.703125, + "learning_rate": 5e-06, + "loss": 0.0005718362517654896, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00057, + "step": 328, + "tokens/total": 7178728, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.015380859375, + "learning_rate": 5e-06, + "loss": 2.0174309611320496e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 329, + "tokens/total": 7198030, + "tokens/train_per_sec_per_gpu": 5.3, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.0380859375, + "learning_rate": 5e-06, + "loss": 3.449153155088425e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 330, + "tokens/total": 7219460, + "tokens/train_per_sec_per_gpu": 5.74, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 3.640625, + "learning_rate": 5e-06, + "loss": 0.0020994965452700853, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0021, + "step": 331, + "tokens/total": 7242092, + "tokens/train_per_sec_per_gpu": 4.53, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.01409912109375, + "learning_rate": 5e-06, + "loss": 1.998012885451317e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 332, + "tokens/total": 7265671, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.3203125, + "learning_rate": 5e-06, + "loss": 0.00019179796800017357, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00019, + "step": 333, + "tokens/total": 7289624, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.232421875, + "learning_rate": 5e-06, + "loss": 5.4980628192424774e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 334, + "tokens/total": 7309312, + "tokens/train_per_sec_per_gpu": 5.19, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 6.5, + "learning_rate": 5e-06, + "loss": 0.003868045751005411, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00388, + "step": 335, + "tokens/total": 7331705, + "tokens/train_per_sec_per_gpu": 4.01, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.369140625, + "learning_rate": 5e-06, + "loss": 0.0003156573511660099, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00032, + "step": 336, + "tokens/total": 7352884, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0478515625, + "learning_rate": 5e-06, + "loss": 5.252566188573837e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 337, + "tokens/total": 7377015, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.06396484375, + "learning_rate": 5e-06, + "loss": 9.608734399080276e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0001, + "step": 338, + "tokens/total": 7399639, + "tokens/train_per_sec_per_gpu": 4.53, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.005340576171875, + "learning_rate": 5e-06, + "loss": 1.0505784302949905e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 339, + "tokens/total": 7421346, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0084228515625, + "learning_rate": 5e-06, + "loss": 1.4592893421649933e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 340, + "tokens/total": 7442997, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.10400390625, + "learning_rate": 5e-06, + "loss": 5.7199038565158844e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 341, + "tokens/total": 7466557, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.0216064453125, + "learning_rate": 5e-06, + "loss": 1.7014332115650177e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 342, + "tokens/total": 7486014, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 2.734375, + "learning_rate": 5e-06, + "loss": 0.0007631033658981323, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00076, + "step": 343, + "tokens/total": 7507954, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.016845703125, + "learning_rate": 5e-06, + "loss": 1.712702214717865e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 344, + "tokens/total": 7530586, + "tokens/train_per_sec_per_gpu": 3.67, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0120849609375, + "learning_rate": 5e-06, + "loss": 2.1358951926231384e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 345, + "tokens/total": 7554320, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.0966796875, + "learning_rate": 5e-06, + "loss": 7.58159440010786e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00008, + "step": 346, + "tokens/total": 7577161, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.004150390625, + "learning_rate": 5e-06, + "loss": 1.1445721611380577e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 347, + "tokens/total": 7599579, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 4.3125, + "learning_rate": 5e-06, + "loss": 0.0015925797633826733, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00159, + "step": 348, + "tokens/total": 7621915, + "tokens/train_per_sec_per_gpu": 5.2, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.046875, + "learning_rate": 5e-06, + "loss": 3.747548907995224e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 349, + "tokens/total": 7642323, + "tokens/train_per_sec_per_gpu": 5.09, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 8.25, + "learning_rate": 5e-06, + "loss": 0.01130049116909504, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01136, + "step": 350, + "tokens/total": 7664661, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0042724609375, + "learning_rate": 5e-06, + "loss": 1.1314405128359795e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 351, + "tokens/total": 7686130, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.004913330078125, + "learning_rate": 5e-06, + "loss": 9.692041203379631e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 352, + "tokens/total": 7708704, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.01519775390625, + "learning_rate": 5e-06, + "loss": 6.9979578256607056e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 353, + "tokens/total": 7731069, + "tokens/train_per_sec_per_gpu": 4.05, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.1025390625, + "learning_rate": 5e-06, + "loss": 5.8176228776574135e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 354, + "tokens/total": 7752662, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.057861328125, + "learning_rate": 5e-06, + "loss": 5.8248406276106834e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 355, + "tokens/total": 7773788, + "tokens/train_per_sec_per_gpu": 5.09, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004302978515625, + "learning_rate": 5e-06, + "loss": 1.0453863069415092e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 356, + "tokens/total": 7796284, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 8.125, + "learning_rate": 5e-06, + "loss": 0.007003905717283487, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00703, + "step": 357, + "tokens/total": 7818437, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.004425048828125, + "learning_rate": 5e-06, + "loss": 8.113216608762741e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 358, + "tokens/total": 7840008, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.047119140625, + "learning_rate": 5e-06, + "loss": 2.065766602754593e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 359, + "tokens/total": 7862878, + "tokens/train_per_sec_per_gpu": 4.52, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.173828125, + "learning_rate": 5e-06, + "loss": 8.578691631555557e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00009, + "step": 360, + "tokens/total": 7885414, + "tokens/train_per_sec_per_gpu": 4.42, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.01483154296875, + "learning_rate": 5e-06, + "loss": 2.0666979253292084e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 361, + "tokens/total": 7907071, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.0869140625, + "learning_rate": 5e-06, + "loss": 4.914449527859688e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 362, + "tokens/total": 7927927, + "tokens/train_per_sec_per_gpu": 4.76, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 1.390625, + "learning_rate": 5e-06, + "loss": 1.6606878489255905e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 363, + "tokens/total": 7950697, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 12.125, + "learning_rate": 5e-06, + "loss": 0.005411576479673386, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00543, + "step": 364, + "tokens/total": 7973328, + "tokens/train_per_sec_per_gpu": 5.7, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 3.75, + "learning_rate": 5e-06, + "loss": 0.0032853991724550724, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00329, + "step": 365, + "tokens/total": 7996123, + "tokens/train_per_sec_per_gpu": 4.96, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.01055908203125, + "learning_rate": 5e-06, + "loss": 1.1832569725811481e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 366, + "tokens/total": 8016666, + "tokens/train_per_sec_per_gpu": 3.99, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.28515625, + "learning_rate": 5e-06, + "loss": 0.00012455368414521217, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00012, + "step": 367, + "tokens/total": 8038081, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.087890625, + "learning_rate": 5e-06, + "loss": 2.0285602658987045e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 368, + "tokens/total": 8058215, + "tokens/train_per_sec_per_gpu": 3.24, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.0128173828125, + "learning_rate": 5e-06, + "loss": 1.985696144402027e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 369, + "tokens/total": 8080480, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0025482177734375, + "learning_rate": 5e-06, + "loss": 5.88572584092617e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 370, + "tokens/total": 8101076, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.039794921875, + "learning_rate": 5e-06, + "loss": 2.5155488401651382e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 371, + "tokens/total": 8123506, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.054931640625, + "learning_rate": 5e-06, + "loss": 4.3221283704042435e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 372, + "tokens/total": 8145338, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 5.625, + "learning_rate": 5e-06, + "loss": 0.0109332837164402, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01099, + "step": 373, + "tokens/total": 8168341, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.13671875, + "learning_rate": 5e-06, + "loss": 7.275678217411041e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 374, + "tokens/total": 8190966, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 4.5625, + "learning_rate": 5e-06, + "loss": 0.004508938640356064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00452, + "step": 375, + "tokens/total": 8213258, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.5546875, + "learning_rate": 5e-06, + "loss": 0.00023284251801669598, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00023, + "step": 376, + "tokens/total": 8236623, + "tokens/train_per_sec_per_gpu": 5.62, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 6.9375, + "learning_rate": 5e-06, + "loss": 0.002499605529010296, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0025, + "step": 377, + "tokens/total": 8259001, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 2.875, + "learning_rate": 5e-06, + "loss": 0.0011327979154884815, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00113, + "step": 378, + "tokens/total": 8280917, + "tokens/train_per_sec_per_gpu": 5.11, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 29.125, + "learning_rate": 5e-06, + "loss": 0.012451283633708954, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01253, + "step": 379, + "tokens/total": 8302787, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.00567626953125, + "learning_rate": 5e-06, + "loss": 7.965369150042534e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 380, + "tokens/total": 8325089, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.76171875, + "learning_rate": 5e-06, + "loss": 0.00017453916370868683, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00017, + "step": 381, + "tokens/total": 8346818, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.027099609375, + "learning_rate": 5e-06, + "loss": 1.0313699021935463e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 382, + "tokens/total": 8366536, + "tokens/train_per_sec_per_gpu": 3.99, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.1015625, + "learning_rate": 5e-06, + "loss": 3.957841545343399e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 383, + "tokens/total": 8387755, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.265625, + "learning_rate": 5e-06, + "loss": 0.0001691377256065607, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00017, + "step": 384, + "tokens/total": 8409959, + "tokens/train_per_sec_per_gpu": 4.96, + "tokens/trainable": 175586 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.298828125, + "learning_rate": 5e-06, + "loss": 0.00016983621753752232, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00017, + "step": 385, + "tokens/total": 8431457, + "tokens/train_per_sec_per_gpu": 3.47, + "tokens/trainable": 176026 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.6484375, + "learning_rate": 5e-06, + "loss": 0.00024128565564751625, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00024, + "step": 386, + "tokens/total": 8454966, + "tokens/train_per_sec_per_gpu": 3.48, + "tokens/trainable": 176512 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.10009765625, + "learning_rate": 5e-06, + "loss": 5.893642082810402e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 387, + "tokens/total": 8475455, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 176943 + }, + { + "epoch": 1.515625, + "grad_norm": 0.005523681640625, + "learning_rate": 5e-06, + "loss": 1.1028721928596497e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 388, + "tokens/total": 8496121, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 177370 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.06787109375, + "learning_rate": 5e-06, + "loss": 5.897274240851402e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 389, + "tokens/total": 8519091, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 177851 + }, + { + "epoch": 1.5234375, + "grad_norm": 2.34375, + "learning_rate": 5e-06, + "loss": 0.0006487315986305475, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00065, + "step": 390, + "tokens/total": 8540554, + "tokens/train_per_sec_per_gpu": 4.62, + "tokens/trainable": 178310 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.044921875, + "learning_rate": 5e-06, + "loss": 3.0490336939692497e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 391, + "tokens/total": 8560847, + "tokens/train_per_sec_per_gpu": 5.37, + "tokens/trainable": 178736 + }, + { + "epoch": 1.53125, + "grad_norm": 0.12158203125, + "learning_rate": 5e-06, + "loss": 0.00010893051512539387, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00011, + "step": 392, + "tokens/total": 8584341, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 179233 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.419921875, + "learning_rate": 5e-06, + "loss": 0.00015373039059340954, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00015, + "step": 393, + "tokens/total": 8607915, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 179730 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.0218505859375, + "learning_rate": 5e-06, + "loss": 2.4413689970970154e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 394, + "tokens/total": 8631883, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 180227 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.138671875, + "learning_rate": 5e-06, + "loss": 6.849993951618671e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 395, + "tokens/total": 8654001, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 180687 + }, + { + "epoch": 1.546875, + "grad_norm": 0.30078125, + "learning_rate": 5e-06, + "loss": 0.00014285719953477383, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00014, + "step": 396, + "tokens/total": 8675833, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 181141 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.287109375, + "learning_rate": 5e-06, + "loss": 0.00015220721252262592, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00015, + "step": 397, + "tokens/total": 8700433, + "tokens/train_per_sec_per_gpu": 5.5, + "tokens/trainable": 181648 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.01129150390625, + "learning_rate": 5e-06, + "loss": 1.4351913705468178e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 398, + "tokens/total": 8722759, + "tokens/train_per_sec_per_gpu": 4.94, + "tokens/trainable": 182109 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.006439208984375, + "learning_rate": 5e-06, + "loss": 8.22986476123333e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 399, + "tokens/total": 8744123, + "tokens/train_per_sec_per_gpu": 3.82, + "tokens/trainable": 182559 + }, + { + "epoch": 1.5625, + "grad_norm": 0.0262451171875, + "learning_rate": 5e-06, + "loss": 1.1917436495423317e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 400, + "tokens/total": 8764941, + "tokens/train_per_sec_per_gpu": 3.86, + "tokens/trainable": 182985 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.02978515625, + "learning_rate": 5e-06, + "loss": 1.268112100660801e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 401, + "tokens/total": 8786988, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 183440 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.007720947265625, + "learning_rate": 5e-06, + "loss": 1.0858988389372826e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 402, + "tokens/total": 8809244, + "tokens/train_per_sec_per_gpu": 4.61, + "tokens/trainable": 183917 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.0074462890625, + "learning_rate": 5e-06, + "loss": 1.249159686267376e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 403, + "tokens/total": 8829849, + "tokens/train_per_sec_per_gpu": 3.93, + "tokens/trainable": 184348 + }, + { + "epoch": 1.578125, + "grad_norm": 0.00830078125, + "learning_rate": 5e-06, + "loss": 1.0677147656679153e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 404, + "tokens/total": 8851196, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 184786 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.04638671875, + "learning_rate": 5e-06, + "loss": 2.430938184261322e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 405, + "tokens/total": 8873641, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 185249 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.07861328125, + "learning_rate": 5e-06, + "loss": 5.474803037941456e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 406, + "tokens/total": 8894869, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 185680 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.0308837890625, + "learning_rate": 5e-06, + "loss": 8.63545574247837e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 407, + "tokens/total": 8915686, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 186100 + }, + { + "epoch": 1.59375, + "grad_norm": 0.002685546875, + "learning_rate": 5e-06, + "loss": 6.081769242882729e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 408, + "tokens/total": 8937519, + "tokens/train_per_sec_per_gpu": 6.0, + "tokens/trainable": 186565 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.1630859375, + "learning_rate": 5e-06, + "loss": 3.738328814506531e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 409, + "tokens/total": 8959352, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 187017 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.01153564453125, + "learning_rate": 5e-06, + "loss": 1.3698241673409939e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 410, + "tokens/total": 8980893, + "tokens/train_per_sec_per_gpu": 4.57, + "tokens/trainable": 187468 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.10205078125, + "learning_rate": 5e-06, + "loss": 4.6446919441223145e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 411, + "tokens/total": 9003759, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 187961 + }, + { + "epoch": 1.609375, + "grad_norm": 0.00579833984375, + "learning_rate": 5e-06, + "loss": 7.294118404388428e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 412, + "tokens/total": 9026963, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 188447 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.004302978515625, + "learning_rate": 5e-06, + "loss": 6.249640136957169e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 413, + "tokens/total": 9048474, + "tokens/train_per_sec_per_gpu": 5.77, + "tokens/trainable": 188903 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.029296875, + "learning_rate": 5e-06, + "loss": 3.9172242395579815e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 414, + "tokens/total": 9071561, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 189398 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.0174560546875, + "learning_rate": 5e-06, + "loss": 1.466367393732071e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 415, + "tokens/total": 9093224, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 189850 + }, + { + "epoch": 1.625, + "grad_norm": 2.59375, + "learning_rate": 5e-06, + "loss": 0.0005653316620737314, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00057, + "step": 416, + "tokens/total": 9115334, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 190306 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.01177978515625, + "learning_rate": 5e-06, + "loss": 7.763737812638283e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 417, + "tokens/total": 9136362, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 190746 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.0133056640625, + "learning_rate": 5e-06, + "loss": 8.842675015330315e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 418, + "tokens/total": 9155868, + "tokens/train_per_sec_per_gpu": 3.62, + "tokens/trainable": 191156 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.03662109375, + "learning_rate": 5e-06, + "loss": 1.9568251445889473e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 419, + "tokens/total": 9176139, + "tokens/train_per_sec_per_gpu": 3.04, + "tokens/trainable": 191573 + }, + { + "epoch": 1.640625, + "grad_norm": 0.0130615234375, + "learning_rate": 5e-06, + "loss": 1.4475896023213863e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 420, + "tokens/total": 9197547, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 192014 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.39453125, + "learning_rate": 5e-06, + "loss": 0.00022255012299865484, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00022, + "step": 421, + "tokens/total": 9218768, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 192453 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.00390625, + "learning_rate": 5e-06, + "loss": 4.441826604306698e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 422, + "tokens/total": 9238601, + "tokens/train_per_sec_per_gpu": 5.48, + "tokens/trainable": 192886 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.002685546875, + "learning_rate": 5e-06, + "loss": 5.251727998256683e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 423, + "tokens/total": 9259813, + "tokens/train_per_sec_per_gpu": 3.74, + "tokens/trainable": 193323 + }, + { + "epoch": 1.65625, + "grad_norm": 0.171875, + "learning_rate": 5e-06, + "loss": 1.033814623951912e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 424, + "tokens/total": 9280479, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 193753 + }, + { + "epoch": 1.66015625, + "grad_norm": 28.625, + "learning_rate": 5e-06, + "loss": 0.00545775331556797, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00547, + "step": 425, + "tokens/total": 9303462, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 194240 + }, + { + "epoch": 1.6640625, + "grad_norm": 5.28125, + "learning_rate": 5e-06, + "loss": 0.008139057084918022, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00817, + "step": 426, + "tokens/total": 9324803, + "tokens/train_per_sec_per_gpu": 4.56, + "tokens/trainable": 194680 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.0093994140625, + "learning_rate": 5e-06, + "loss": 8.708564564585686e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 427, + "tokens/total": 9348550, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 195186 + }, + { + "epoch": 1.671875, + "grad_norm": 0.0174560546875, + "learning_rate": 5e-06, + "loss": 1.189298927783966e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 428, + "tokens/total": 9373086, + "tokens/train_per_sec_per_gpu": 4.41, + "tokens/trainable": 195693 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.046142578125, + "learning_rate": 5e-06, + "loss": 2.0359992049634457e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 429, + "tokens/total": 9393647, + "tokens/train_per_sec_per_gpu": 4.62, + "tokens/trainable": 196130 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.01251220703125, + "learning_rate": 5e-06, + "loss": 5.70458360016346e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 430, + "tokens/total": 9412748, + "tokens/train_per_sec_per_gpu": 3.38, + "tokens/trainable": 196530 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.01104736328125, + "learning_rate": 5e-06, + "loss": 1.2532109394669533e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 431, + "tokens/total": 9434167, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 196974 + }, + { + "epoch": 1.6875, + "grad_norm": 0.0208740234375, + "learning_rate": 5e-06, + "loss": 1.5752622857689857e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 432, + "tokens/total": 9457579, + "tokens/train_per_sec_per_gpu": 4.27, + "tokens/trainable": 197454 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.0031280517578125, + "learning_rate": 5e-06, + "loss": 5.261856131255627e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 433, + "tokens/total": 9478689, + "tokens/train_per_sec_per_gpu": 3.12, + "tokens/trainable": 197902 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.0027923583984375, + "learning_rate": 5e-06, + "loss": 5.483860149979591e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 434, + "tokens/total": 9501276, + "tokens/train_per_sec_per_gpu": 5.08, + "tokens/trainable": 198374 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.0118408203125, + "learning_rate": 5e-06, + "loss": 1.2922100722789764e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 435, + "tokens/total": 9523960, + "tokens/train_per_sec_per_gpu": 4.92, + "tokens/trainable": 198846 + }, + { + "epoch": 1.703125, + "grad_norm": 3.0, + "learning_rate": 5e-06, + "loss": 0.0016529926797375083, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00165, + "step": 436, + "tokens/total": 9546519, + "tokens/train_per_sec_per_gpu": 5.69, + "tokens/trainable": 199320 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.01513671875, + "learning_rate": 5e-06, + "loss": 1.334841363132e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 437, + "tokens/total": 9568310, + "tokens/train_per_sec_per_gpu": 5.28, + "tokens/trainable": 199777 + }, + { + "epoch": 1.7109375, + "grad_norm": 0.244140625, + "learning_rate": 5e-06, + "loss": 3.59085388481617e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 438, + "tokens/total": 9587812, + "tokens/train_per_sec_per_gpu": 4.02, + "tokens/trainable": 200194 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.00689697265625, + "learning_rate": 5e-06, + "loss": 8.972478099167347e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 439, + "tokens/total": 9610174, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 200663 + }, + { + "epoch": 1.71875, + "grad_norm": 0.10791015625, + "learning_rate": 5e-06, + "loss": 5.8091478422284126e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 440, + "tokens/total": 9632002, + "tokens/train_per_sec_per_gpu": 5.06, + "tokens/trainable": 201121 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.06640625, + "learning_rate": 5e-06, + "loss": 2.7986010536551476e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 441, + "tokens/total": 9654521, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 201598 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.2021484375, + "learning_rate": 5e-06, + "loss": 5.4487958550453186e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 442, + "tokens/total": 9676426, + "tokens/train_per_sec_per_gpu": 5.1, + "tokens/trainable": 202068 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.0703125, + "learning_rate": 5e-06, + "loss": 3.0130031518638134e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 443, + "tokens/total": 9698234, + "tokens/train_per_sec_per_gpu": 4.59, + "tokens/trainable": 202524 + }, + { + "epoch": 1.734375, + "grad_norm": 0.1572265625, + "learning_rate": 5e-06, + "loss": 6.322888657450676e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 444, + "tokens/total": 9720232, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 202996 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.0771484375, + "learning_rate": 5e-06, + "loss": 6.508128717541695e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00007, + "step": 445, + "tokens/total": 9742377, + "tokens/train_per_sec_per_gpu": 4.62, + "tokens/trainable": 203470 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.00360107421875, + "learning_rate": 5e-06, + "loss": 6.959191523492336e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 446, + "tokens/total": 9764493, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 203919 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.0234375, + "learning_rate": 5e-06, + "loss": 1.2123840861022472e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 447, + "tokens/total": 9786740, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 204376 + }, + { + "epoch": 1.75, + "grad_norm": 0.0615234375, + "learning_rate": 5e-06, + "loss": 3.348151221871376e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 448, + "tokens/total": 9808148, + "tokens/train_per_sec_per_gpu": 4.7, + "tokens/trainable": 204821 + }, + { + "epoch": 1.75390625, + "grad_norm": 0.006072998046875, + "learning_rate": 5e-06, + "loss": 9.379931725561619e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 449, + "tokens/total": 9829386, + "tokens/train_per_sec_per_gpu": 4.37, + "tokens/trainable": 205271 + }, + { + "epoch": 1.7578125, + "grad_norm": 0.040771484375, + "learning_rate": 5e-06, + "loss": 1.0490999557077885e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 450, + "tokens/total": 9851143, + "tokens/train_per_sec_per_gpu": 3.3, + "tokens/trainable": 205723 + }, + { + "epoch": 1.76171875, + "grad_norm": 0.1064453125, + "learning_rate": 5e-06, + "loss": 4.542467650026083e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00005, + "step": 451, + "tokens/total": 9872664, + "tokens/train_per_sec_per_gpu": 4.61, + "tokens/trainable": 206169 + }, + { + "epoch": 1.765625, + "grad_norm": 0.03857421875, + "learning_rate": 5e-06, + "loss": 9.22114122658968e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 452, + "tokens/total": 9892881, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 206592 + }, + { + "epoch": 1.76953125, + "grad_norm": 0.005859375, + "learning_rate": 5e-06, + "loss": 9.221024811267853e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 453, + "tokens/total": 9914682, + "tokens/train_per_sec_per_gpu": 4.77, + "tokens/trainable": 207055 + }, + { + "epoch": 1.7734375, + "grad_norm": 0.01251220703125, + "learning_rate": 5e-06, + "loss": 1.1057360097765923e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 454, + "tokens/total": 9935096, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 207494 + }, + { + "epoch": 1.77734375, + "grad_norm": 0.00286865234375, + "learning_rate": 5e-06, + "loss": 4.5412452891469e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 455, + "tokens/total": 9956837, + "tokens/train_per_sec_per_gpu": 4.53, + "tokens/trainable": 207956 + }, + { + "epoch": 1.78125, + "grad_norm": 0.006622314453125, + "learning_rate": 5e-06, + "loss": 6.343703716993332e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 456, + "tokens/total": 9978343, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 208408 + }, + { + "epoch": 1.78515625, + "grad_norm": 0.1640625, + "learning_rate": 5e-06, + "loss": 6.084947381168604e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00006, + "step": 457, + "tokens/total": 10001764, + "tokens/train_per_sec_per_gpu": 4.95, + "tokens/trainable": 208889 + }, + { + "epoch": 1.7890625, + "grad_norm": 0.03173828125, + "learning_rate": 5e-06, + "loss": 1.0989373549818993e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 458, + "tokens/total": 10022890, + "tokens/train_per_sec_per_gpu": 5.18, + "tokens/trainable": 209336 + }, + { + "epoch": 1.79296875, + "grad_norm": 0.0242919921875, + "learning_rate": 5e-06, + "loss": 1.9826809875667095e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 459, + "tokens/total": 10044939, + "tokens/train_per_sec_per_gpu": 5.04, + "tokens/trainable": 209797 + }, + { + "epoch": 1.796875, + "grad_norm": 1.28125, + "learning_rate": 5e-06, + "loss": 0.000303889624774456, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0003, + "step": 460, + "tokens/total": 10066429, + "tokens/train_per_sec_per_gpu": 3.97, + "tokens/trainable": 210240 + }, + { + "epoch": 1.80078125, + "grad_norm": 0.0556640625, + "learning_rate": 5e-06, + "loss": 4.40418953076005e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 461, + "tokens/total": 10088503, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 210702 + }, + { + "epoch": 1.8046875, + "grad_norm": 0.0108642578125, + "learning_rate": 5e-06, + "loss": 9.459559805691242e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 462, + "tokens/total": 10109108, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 211140 + }, + { + "epoch": 1.80859375, + "grad_norm": 2.015625, + "learning_rate": 5e-06, + "loss": 0.00018418568652123213, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00018, + "step": 463, + "tokens/total": 10130403, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 211575 + }, + { + "epoch": 1.8125, + "grad_norm": 0.022705078125, + "learning_rate": 5e-06, + "loss": 1.1794967576861382e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 464, + "tokens/total": 10154368, + "tokens/train_per_sec_per_gpu": 5.74, + "tokens/trainable": 212083 + }, + { + "epoch": 1.81640625, + "grad_norm": 0.01507568359375, + "learning_rate": 5e-06, + "loss": 1.228495966643095e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 465, + "tokens/total": 10177259, + "tokens/train_per_sec_per_gpu": 3.71, + "tokens/trainable": 212558 + }, + { + "epoch": 1.8203125, + "grad_norm": 0.06494140625, + "learning_rate": 5e-06, + "loss": 2.8050155378878117e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 466, + "tokens/total": 10198465, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 213014 + }, + { + "epoch": 1.82421875, + "grad_norm": 0.0023040771484375, + "learning_rate": 5e-06, + "loss": 4.879082553088665e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 467, + "tokens/total": 10220237, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 213465 + }, + { + "epoch": 1.828125, + "grad_norm": 6.59375, + "learning_rate": 5e-06, + "loss": 0.0018728224094957113, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00187, + "step": 468, + "tokens/total": 10242601, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 213940 + }, + { + "epoch": 1.83203125, + "grad_norm": 0.003570556640625, + "learning_rate": 5e-06, + "loss": 3.997585736215115e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 469, + "tokens/total": 10265523, + "tokens/train_per_sec_per_gpu": 4.34, + "tokens/trainable": 214415 + }, + { + "epoch": 1.8359375, + "grad_norm": 0.0146484375, + "learning_rate": 5e-06, + "loss": 1.2936885468661785e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 470, + "tokens/total": 10287850, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 214880 + }, + { + "epoch": 1.83984375, + "grad_norm": 0.0162353515625, + "learning_rate": 5e-06, + "loss": 1.1437805369496346e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 471, + "tokens/total": 10308946, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 215334 + }, + { + "epoch": 1.84375, + "grad_norm": 0.0021514892578125, + "learning_rate": 5e-06, + "loss": 3.0597439035773277e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 472, + "tokens/total": 10332114, + "tokens/train_per_sec_per_gpu": 5.75, + "tokens/trainable": 215822 + }, + { + "epoch": 1.84765625, + "grad_norm": 0.02294921875, + "learning_rate": 5e-06, + "loss": 1.649046316742897e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 473, + "tokens/total": 10351875, + "tokens/train_per_sec_per_gpu": 4.72, + "tokens/trainable": 216239 + }, + { + "epoch": 1.8515625, + "grad_norm": 0.00665283203125, + "learning_rate": 5e-06, + "loss": 6.865535397082567e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 474, + "tokens/total": 10373073, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 216691 + }, + { + "epoch": 1.85546875, + "grad_norm": 0.0036163330078125, + "learning_rate": 5e-06, + "loss": 5.9650628827512264e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 475, + "tokens/total": 10395433, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 217162 + }, + { + "epoch": 1.859375, + "grad_norm": 0.0133056640625, + "learning_rate": 5e-06, + "loss": 9.11857932806015e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 476, + "tokens/total": 10418883, + "tokens/train_per_sec_per_gpu": 4.3, + "tokens/trainable": 217638 + }, + { + "epoch": 1.86328125, + "grad_norm": 0.0125732421875, + "learning_rate": 5e-06, + "loss": 2.887565642595291e-06, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 477, + "tokens/total": 10438781, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 218053 + }, + { + "epoch": 1.8671875, + "grad_norm": 0.07763671875, + "learning_rate": 5e-06, + "loss": 2.127542393282056e-05, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 478, + "tokens/total": 10460131, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 218503 + }, + { + "epoch": 1.87109375, + "grad_norm": 0.33203125, + "learning_rate": 5e-06, + "loss": 0.0001157271908596158, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00012, + "step": 479, + "tokens/total": 10482563, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 218964 + }, + { + "epoch": 1.875, + "grad_norm": 0.006561279296875, + "learning_rate": 5e-06, + "loss": 7.29516614228487e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 480, + "tokens/total": 10503663, + "tokens/train_per_sec_per_gpu": 5.85, + "tokens/trainable": 219412 + }, + { + "epoch": 1.87890625, + "grad_norm": 0.0081787109375, + "learning_rate": 5e-06, + "loss": 5.040958058089018e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 481, + "tokens/total": 10526070, + "tokens/train_per_sec_per_gpu": 5.77, + "tokens/trainable": 219886 + }, + { + "epoch": 1.8828125, + "grad_norm": 0.01214599609375, + "learning_rate": 5e-06, + "loss": 1.437158789485693e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 482, + "tokens/total": 10547791, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 220343 + }, + { + "epoch": 1.88671875, + "grad_norm": 0.004425048828125, + "learning_rate": 5e-06, + "loss": 5.447596777230501e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 483, + "tokens/total": 10571096, + "tokens/train_per_sec_per_gpu": 4.31, + "tokens/trainable": 220825 + }, + { + "epoch": 1.890625, + "grad_norm": 0.369140625, + "learning_rate": 5e-06, + "loss": 0.00015859410632401705, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00016, + "step": 484, + "tokens/total": 10593446, + "tokens/train_per_sec_per_gpu": 3.8, + "tokens/trainable": 221282 + }, + { + "epoch": 1.89453125, + "grad_norm": 0.01025390625, + "learning_rate": 5e-06, + "loss": 5.2228569984436035e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 485, + "tokens/total": 10617720, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 221780 + }, + { + "epoch": 1.8984375, + "grad_norm": 0.007781982421875, + "learning_rate": 5e-06, + "loss": 8.30693170428276e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 486, + "tokens/total": 10637793, + "tokens/train_per_sec_per_gpu": 3.69, + "tokens/trainable": 222210 + }, + { + "epoch": 1.90234375, + "grad_norm": 5.03125, + "learning_rate": 5e-06, + "loss": 0.00522148422896862, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00524, + "step": 487, + "tokens/total": 10659041, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 222646 + }, + { + "epoch": 1.90625, + "grad_norm": 5.5625, + "learning_rate": 5e-06, + "loss": 0.001833226066082716, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00183, + "step": 488, + "tokens/total": 10679287, + "tokens/train_per_sec_per_gpu": 4.1, + "tokens/trainable": 223066 + }, + { + "epoch": 1.91015625, + "grad_norm": 0.033203125, + "learning_rate": 5e-06, + "loss": 1.1670519597828388e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 489, + "tokens/total": 10700063, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 223512 + }, + { + "epoch": 1.9140625, + "grad_norm": 0.002288818359375, + "learning_rate": 5e-06, + "loss": 3.7804129533469677e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 490, + "tokens/total": 10720492, + "tokens/train_per_sec_per_gpu": 4.12, + "tokens/trainable": 223956 + }, + { + "epoch": 1.91796875, + "grad_norm": 0.0062255859375, + "learning_rate": 5e-06, + "loss": 5.738227628171444e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 491, + "tokens/total": 10742485, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 224416 + }, + { + "epoch": 1.921875, + "grad_norm": 3.5, + "learning_rate": 5e-06, + "loss": 0.0006399974226951599, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00064, + "step": 492, + "tokens/total": 10764717, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 224883 + }, + { + "epoch": 1.92578125, + "grad_norm": 0.00173187255859375, + "learning_rate": 5e-06, + "loss": 3.3015385270118713e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 493, + "tokens/total": 10786238, + "tokens/train_per_sec_per_gpu": 4.49, + "tokens/trainable": 225324 + }, + { + "epoch": 1.9296875, + "grad_norm": 3.21875, + "learning_rate": 5e-06, + "loss": 0.001615986693650484, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00162, + "step": 494, + "tokens/total": 10807783, + "tokens/train_per_sec_per_gpu": 5.8, + "tokens/trainable": 225783 + }, + { + "epoch": 1.93359375, + "grad_norm": 1.609375, + "learning_rate": 5e-06, + "loss": 0.0007133671315386891, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00071, + "step": 495, + "tokens/total": 10831124, + "tokens/train_per_sec_per_gpu": 5.05, + "tokens/trainable": 226281 + }, + { + "epoch": 1.9375, + "grad_norm": 0.0023040771484375, + "learning_rate": 5e-06, + "loss": 3.1259842216968536e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 496, + "tokens/total": 10852782, + "tokens/train_per_sec_per_gpu": 5.2, + "tokens/trainable": 226735 + }, + { + "epoch": 1.94140625, + "grad_norm": 0.005645751953125, + "learning_rate": 5e-06, + "loss": 6.007147021591663e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 497, + "tokens/total": 10874927, + "tokens/train_per_sec_per_gpu": 4.48, + "tokens/trainable": 227198 + }, + { + "epoch": 1.9453125, + "grad_norm": 0.005462646484375, + "learning_rate": 5e-06, + "loss": 3.9049191400408745e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 498, + "tokens/total": 10895243, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 227623 + }, + { + "epoch": 1.94921875, + "grad_norm": 0.01422119140625, + "learning_rate": 5e-06, + "loss": 1.1300435289740562e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 499, + "tokens/total": 10916401, + "tokens/train_per_sec_per_gpu": 4.58, + "tokens/trainable": 228074 + }, + { + "epoch": 1.953125, + "grad_norm": 0.007293701171875, + "learning_rate": 5e-06, + "loss": 9.988900274038315e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 500, + "tokens/total": 10938594, + "tokens/train_per_sec_per_gpu": 3.87, + "tokens/trainable": 228541 + }, + { + "epoch": 1.95703125, + "grad_norm": 8.1875, + "learning_rate": 5e-06, + "loss": 0.001212581293657422, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00121, + "step": 501, + "tokens/total": 10962017, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 229031 + }, + { + "epoch": 1.9609375, + "grad_norm": 0.06689453125, + "learning_rate": 5e-06, + "loss": 3.0907627660781145e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 502, + "tokens/total": 10984837, + "tokens/train_per_sec_per_gpu": 4.39, + "tokens/trainable": 229512 + }, + { + "epoch": 1.96484375, + "grad_norm": 0.06298828125, + "learning_rate": 5e-06, + "loss": 2.287188544869423e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 503, + "tokens/total": 11007761, + "tokens/train_per_sec_per_gpu": 5.02, + "tokens/trainable": 229989 + }, + { + "epoch": 1.96875, + "grad_norm": 0.08203125, + "learning_rate": 5e-06, + "loss": 2.659979509189725e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 504, + "tokens/total": 11030063, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 230447 + }, + { + "epoch": 1.97265625, + "grad_norm": 0.034912109375, + "learning_rate": 5e-06, + "loss": 1.7017940990626812e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00002, + "step": 505, + "tokens/total": 11050931, + "tokens/train_per_sec_per_gpu": 5.22, + "tokens/trainable": 230879 + }, + { + "epoch": 1.9765625, + "grad_norm": 0.08935546875, + "learning_rate": 5e-06, + "loss": 2.7938978746533394e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00003, + "step": 506, + "tokens/total": 11073908, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 231348 + }, + { + "epoch": 1.98046875, + "grad_norm": 1.7578125, + "learning_rate": 5e-06, + "loss": 0.0007111791055649519, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00071, + "step": 507, + "tokens/total": 11096475, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 231819 + }, + { + "epoch": 1.984375, + "grad_norm": 0.0260009765625, + "learning_rate": 5e-06, + "loss": 9.796989616006613e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00001, + "step": 508, + "tokens/total": 11118068, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 232269 + }, + { + "epoch": 1.98828125, + "grad_norm": 0.0517578125, + "learning_rate": 5e-06, + "loss": 3.949366509914398e-05, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00004, + "step": 509, + "tokens/total": 11139928, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 232716 + }, + { + "epoch": 1.9921875, + "grad_norm": 0.00787353515625, + "learning_rate": 5e-06, + "loss": 3.902765456587076e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 510, + "tokens/total": 11162210, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 233178 + }, + { + "epoch": 1.99609375, + "grad_norm": 1.5859375, + "learning_rate": 5e-06, + "loss": 0.0007482740329578519, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00075, + "step": 511, + "tokens/total": 11184282, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 233647 + }, + { + "epoch": 2.0, + "grad_norm": 0.005126953125, + "learning_rate": 5e-06, + "loss": 3.1841336749494076e-06, + "memory/device_reserved (GiB)": 60.16, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0, + "step": 512, + "tokens/total": 11204672, + "tokens/train_per_sec_per_gpu": 3.11, + "tokens/trainable": 234060 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 7.513221816983224e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-512/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-512/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-512/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-64/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-64/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/config.json b/full_aft_midtrain4/balanced/checkpoint-64/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-64/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-64/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..33df9f18dede183408a260a8d2b240dab89e184e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c2ad549c6bd04acaeb10fcb36376e837e3d10d6f884b1f84c70f00bac7fabcd +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-64/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-64/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-64/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-64/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-64/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-64/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-64/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cce8307209eeee36e580a09870a1cba7e9b4ee1f --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/tokens_state.json @@ -0,0 +1 @@ +{"total": 1399140, "trainable": 29283} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-64/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-64/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3e8490e8579c77f39d96ff93812f09e6eaf5b400 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/trainer_state.json @@ -0,0 +1,930 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.25, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 13.5625, + "learning_rate": 5e-06, + "loss": 0.05783689022064209, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05954, + "step": 9, + "tokens/total": 196358, + "tokens/train_per_sec_per_gpu": 3.39, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 16.375, + "learning_rate": 5e-06, + "loss": 0.05202341079711914, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0534, + "step": 10, + "tokens/total": 218622, + "tokens/train_per_sec_per_gpu": 4.33, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 22.5, + "learning_rate": 5e-06, + "loss": 0.03920435905456543, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03998, + "step": 11, + "tokens/total": 239835, + "tokens/train_per_sec_per_gpu": 4.54, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 43.75, + "learning_rate": 5e-06, + "loss": 0.0486750602722168, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04988, + "step": 12, + "tokens/total": 264428, + "tokens/train_per_sec_per_gpu": 4.93, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 7.59375, + "learning_rate": 5e-06, + "loss": 0.023757457733154297, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02404, + "step": 13, + "tokens/total": 284859, + "tokens/train_per_sec_per_gpu": 3.95, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 19.0, + "learning_rate": 5e-06, + "loss": 0.04330146312713623, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04425, + "step": 14, + "tokens/total": 305640, + "tokens/train_per_sec_per_gpu": 3.17, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 9.6875, + "learning_rate": 5e-06, + "loss": 0.03816962242126465, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03891, + "step": 15, + "tokens/total": 327063, + "tokens/train_per_sec_per_gpu": 4.35, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 11.8125, + "learning_rate": 5e-06, + "loss": 0.03375864028930664, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03433, + "step": 16, + "tokens/total": 347838, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 12.4375, + "learning_rate": 5e-06, + "loss": 0.023131251335144043, + "memory/device_reserved (GiB)": 60.1, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0234, + "step": 17, + "tokens/total": 369888, + "tokens/train_per_sec_per_gpu": 4.4, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 10.75, + "learning_rate": 5e-06, + "loss": 0.02522754669189453, + "memory/device_reserved (GiB)": 57.25, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02555, + "step": 18, + "tokens/total": 392455, + "tokens/train_per_sec_per_gpu": 3.72, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 8.4375, + "learning_rate": 5e-06, + "loss": 0.028311967849731445, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02872, + "step": 19, + "tokens/total": 413549, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 4.34375, + "learning_rate": 5e-06, + "loss": 0.012633323669433594, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01271, + "step": 20, + "tokens/total": 434128, + "tokens/train_per_sec_per_gpu": 3.7, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 3.34375, + "learning_rate": 5e-06, + "loss": 0.010331928730010986, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01039, + "step": 21, + "tokens/total": 456131, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 8.6875, + "learning_rate": 5e-06, + "loss": 0.011040687561035156, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0111, + "step": 22, + "tokens/total": 478707, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 17.75, + "learning_rate": 5e-06, + "loss": 0.030008912086486816, + "memory/device_reserved (GiB)": 58.4, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03046, + "step": 23, + "tokens/total": 501425, + "tokens/train_per_sec_per_gpu": 4.9, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 6.6875, + "learning_rate": 5e-06, + "loss": 0.007420599460601807, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00745, + "step": 24, + "tokens/total": 522885, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 24.75, + "learning_rate": 5e-06, + "loss": 0.05265447497367859, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05407, + "step": 25, + "tokens/total": 544886, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 12.625, + "learning_rate": 5e-06, + "loss": 0.04731038212776184, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04845, + "step": 26, + "tokens/total": 564964, + "tokens/train_per_sec_per_gpu": 3.36, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 13.1875, + "learning_rate": 5e-06, + "loss": 0.014921486377716064, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01503, + "step": 27, + "tokens/total": 586567, + "tokens/train_per_sec_per_gpu": 5.29, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 18.625, + "learning_rate": 5e-06, + "loss": 0.013006031513214111, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01309, + "step": 28, + "tokens/total": 608188, + "tokens/train_per_sec_per_gpu": 3.78, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 21.25, + "learning_rate": 5e-06, + "loss": 0.028101369738578796, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0285, + "step": 29, + "tokens/total": 630007, + "tokens/train_per_sec_per_gpu": 3.81, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 8.3125, + "learning_rate": 5e-06, + "loss": 0.01690027117729187, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01704, + "step": 30, + "tokens/total": 651656, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.015246853232383728, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01536, + "step": 31, + "tokens/total": 672938, + "tokens/train_per_sec_per_gpu": 5.14, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 11.125, + "learning_rate": 5e-06, + "loss": 0.017684906721115112, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01784, + "step": 32, + "tokens/total": 696525, + "tokens/train_per_sec_per_gpu": 5.03, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 15.5, + "learning_rate": 5e-06, + "loss": 0.02578166127204895, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02612, + "step": 33, + "tokens/total": 717452, + "tokens/train_per_sec_per_gpu": 2.67, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 10.5625, + "learning_rate": 5e-06, + "loss": 0.03840416669845581, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03915, + "step": 34, + "tokens/total": 738762, + "tokens/train_per_sec_per_gpu": 3.32, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 3.921875, + "learning_rate": 5e-06, + "loss": 0.017821550369262695, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01798, + "step": 35, + "tokens/total": 762239, + "tokens/train_per_sec_per_gpu": 5.21, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 5.71875, + "learning_rate": 5e-06, + "loss": 0.02945883572101593, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0299, + "step": 36, + "tokens/total": 784340, + "tokens/train_per_sec_per_gpu": 3.76, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 24.625, + "learning_rate": 5e-06, + "loss": 0.011434584856033325, + "memory/device_reserved (GiB)": 58.59, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0115, + "step": 37, + "tokens/total": 806003, + "tokens/train_per_sec_per_gpu": 4.55, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 6.71875, + "learning_rate": 5e-06, + "loss": 0.016611874103546143, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01675, + "step": 38, + "tokens/total": 828092, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 5.53125, + "learning_rate": 5e-06, + "loss": 0.012658625841140747, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 39, + "tokens/total": 849305, + "tokens/train_per_sec_per_gpu": 4.44, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 8.8125, + "learning_rate": 5e-06, + "loss": 0.02379515767097473, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02408, + "step": 40, + "tokens/total": 871803, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 2.015625, + "learning_rate": 5e-06, + "loss": 0.0050678253173828125, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00508, + "step": 41, + "tokens/total": 893895, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 5.875, + "learning_rate": 5e-06, + "loss": 0.016993582248687744, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01714, + "step": 42, + "tokens/total": 914803, + "tokens/train_per_sec_per_gpu": 3.89, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 4.1875, + "learning_rate": 5e-06, + "loss": 0.025514155626296997, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02584, + "step": 43, + "tokens/total": 938486, + "tokens/train_per_sec_per_gpu": 4.88, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 10.1875, + "learning_rate": 5e-06, + "loss": 0.013556092977523804, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01365, + "step": 44, + "tokens/total": 961149, + "tokens/train_per_sec_per_gpu": 4.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 3.9375, + "learning_rate": 5e-06, + "loss": 0.012663766741752625, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01274, + "step": 45, + "tokens/total": 982727, + "tokens/train_per_sec_per_gpu": 4.43, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 2.421875, + "learning_rate": 5e-06, + "loss": 0.00803878903388977, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00807, + "step": 46, + "tokens/total": 1003446, + "tokens/train_per_sec_per_gpu": 5.17, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 6.40625, + "learning_rate": 5e-06, + "loss": 0.027271926403045654, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.02765, + "step": 47, + "tokens/total": 1024518, + "tokens/train_per_sec_per_gpu": 3.84, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 6.90625, + "learning_rate": 5e-06, + "loss": 0.019454196095466614, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01964, + "step": 48, + "tokens/total": 1046348, + "tokens/train_per_sec_per_gpu": 3.79, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 14.1875, + "learning_rate": 5e-06, + "loss": 0.007123962044715881, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00715, + "step": 49, + "tokens/total": 1067448, + "tokens/train_per_sec_per_gpu": 4.46, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 6.65625, + "learning_rate": 5e-06, + "loss": 0.016036301851272583, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01617, + "step": 50, + "tokens/total": 1091250, + "tokens/train_per_sec_per_gpu": 5.67, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 6.625, + "learning_rate": 5e-06, + "loss": 0.006014183163642883, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00603, + "step": 51, + "tokens/total": 1112130, + "tokens/train_per_sec_per_gpu": 3.9, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 8.5, + "learning_rate": 5e-06, + "loss": 0.01116333156824112, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01123, + "step": 52, + "tokens/total": 1135386, + "tokens/train_per_sec_per_gpu": 4.38, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 7.53125, + "learning_rate": 5e-06, + "loss": 0.012836582958698273, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01292, + "step": 53, + "tokens/total": 1156121, + "tokens/train_per_sec_per_gpu": 3.88, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 3.453125, + "learning_rate": 5e-06, + "loss": 0.005587518215179443, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.0056, + "step": 54, + "tokens/total": 1176702, + "tokens/train_per_sec_per_gpu": 3.68, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 78.5, + "learning_rate": 5e-06, + "loss": 0.05362102389335632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05508, + "step": 55, + "tokens/total": 1198839, + "tokens/train_per_sec_per_gpu": 3.13, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 2.796875, + "learning_rate": 5e-06, + "loss": 0.005013041198253632, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00503, + "step": 56, + "tokens/total": 1219997, + "tokens/train_per_sec_per_gpu": 4.51, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.65234375, + "learning_rate": 5e-06, + "loss": 0.0010031796991825104, + "memory/device_reserved (GiB)": 60.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.001, + "step": 57, + "tokens/total": 1241006, + "tokens/train_per_sec_per_gpu": 4.5, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 2.1875, + "learning_rate": 5e-06, + "loss": 0.0032418183982372284, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00325, + "step": 58, + "tokens/total": 1263530, + "tokens/train_per_sec_per_gpu": 4.29, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 15.3125, + "learning_rate": 5e-06, + "loss": 0.007594488561153412, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00762, + "step": 59, + "tokens/total": 1287978, + "tokens/train_per_sec_per_gpu": 4.36, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 10.375, + "learning_rate": 5e-06, + "loss": 0.009997941553592682, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.01005, + "step": 60, + "tokens/total": 1309463, + "tokens/train_per_sec_per_gpu": 3.83, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 2.34375, + "learning_rate": 5e-06, + "loss": 0.0018807370215654373, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.00188, + "step": 61, + "tokens/total": 1331283, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 11.6875, + "learning_rate": 5e-06, + "loss": 0.03929818421602249, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.04008, + "step": 62, + "tokens/total": 1352434, + "tokens/train_per_sec_per_gpu": 3.85, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 12.3125, + "learning_rate": 5e-06, + "loss": 0.03207191079854965, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03259, + "step": 63, + "tokens/total": 1376383, + "tokens/train_per_sec_per_gpu": 5.07, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 27.125, + "learning_rate": 5e-06, + "loss": 0.03780709207057953, + "memory/device_reserved (GiB)": 60.14, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03853, + "step": 64, + "tokens/total": 1399140, + "tokens/train_per_sec_per_gpu": 5.15, + "tokens/trainable": 29283 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.381844608037683e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-64/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-64/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-64/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913 diff --git a/full_aft_midtrain4/balanced/checkpoint-8/chat_template.jinja b/full_aft_midtrain4/balanced/checkpoint-8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/config.json b/full_aft_midtrain4/balanced/checkpoint-8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/generation_config.json b/full_aft_midtrain4/balanced/checkpoint-8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..21550a3716be54c59bb9587952923fd89452c359 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.9.0" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/model.safetensors b/full_aft_midtrain4/balanced/checkpoint-8/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b8fc339bb2b793fe409139c19648da5d1dfd8fc --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c657e2a9a2c43ec671670e3ba921cac36187b541507f387531c0b8c08330de2 +size 26388552360 diff --git a/full_aft_midtrain4/balanced/checkpoint-8/preprocessor_config.json b/full_aft_midtrain4/balanced/checkpoint-8/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e00fc184f61b698181821169c6374cd5813e5c --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/preprocessor_config.json @@ -0,0 +1,29 @@ +{ + "do_convert_rgb": null, + "do_normalize": true, + "do_pan_and_scan": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Gemma3ImageProcessor", + "image_seq_length": 256, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "pan_and_scan_max_num_crops": null, + "pan_and_scan_min_crop_size": null, + "pan_and_scan_min_ratio_to_activate": null, + "processor_class": "Gemma3Processor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 896, + "width": 896 + } +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/processor_config.json b/full_aft_midtrain4/balanced/checkpoint-8/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..453c7966d4b5d0b4a317c585989f64c58c2a6bf0 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/processor_config.json @@ -0,0 +1,4 @@ +{ + "image_seq_length": 256, + "processor_class": "Gemma3Processor" +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/tokenizer.json b/full_aft_midtrain4/balanced/checkpoint-8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/full_aft_midtrain4/balanced/checkpoint-8/tokenizer_config.json b/full_aft_midtrain4/balanced/checkpoint-8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/tokens_state.json b/full_aft_midtrain4/balanced/checkpoint-8/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aaae74c7a1432ec07059570f97ac702de21c10ff --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/tokens_state.json @@ -0,0 +1 @@ +{"total": 175713, "trainable": 3655} \ No newline at end of file diff --git a/full_aft_midtrain4/balanced/checkpoint-8/trainer_state.json b/full_aft_midtrain4/balanced/checkpoint-8/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0b3634877401f09d3bde079c7338d8d303f8c1a3 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/trainer_state.json @@ -0,0 +1,146 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.03125, + "eval_steps": 500, + "global_step": 8, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 17.375, + "learning_rate": 5e-06, + "loss": 0.11627960205078125, + "memory/device_reserved (GiB)": 46.13, + "memory/max_active (GiB)": 35.94, + "memory/max_allocated (GiB)": 35.94, + "ppl": 1.12331, + "step": 1, + "tokens/total": 22614, + "tokens/train_per_sec_per_gpu": 1.99, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1472.0, + "learning_rate": 5e-06, + "loss": 0.1474212110042572, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.15884, + "step": 2, + "tokens/total": 44116, + "tokens/train_per_sec_per_gpu": 4.14, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 57.75, + "learning_rate": 5e-06, + "loss": 0.2154746651649475, + "memory/device_reserved (GiB)": 58.08, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.24045, + "step": 3, + "tokens/total": 67092, + "tokens/train_per_sec_per_gpu": 3.77, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 37.75, + "learning_rate": 5e-06, + "loss": 0.10307750105857849, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.10858, + "step": 4, + "tokens/total": 88800, + "tokens/train_per_sec_per_gpu": 5.16, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 25.0, + "learning_rate": 5e-06, + "loss": 0.10710409283638, + "memory/device_reserved (GiB)": 58.09, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.11305, + "step": 5, + "tokens/total": 110742, + "tokens/train_per_sec_per_gpu": 3.92, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 16.25, + "learning_rate": 5e-06, + "loss": 0.05220422148704529, + "memory/device_reserved (GiB)": 57.19, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.05359, + "step": 6, + "tokens/total": 131497, + "tokens/train_per_sec_per_gpu": 4.45, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 18.0, + "learning_rate": 5e-06, + "loss": 0.06188321113586426, + "memory/device_reserved (GiB)": 59.04, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.06384, + "step": 7, + "tokens/total": 153846, + "tokens/train_per_sec_per_gpu": 4.99, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 16.75, + "learning_rate": 5e-06, + "loss": 0.036078453063964844, + "memory/device_reserved (GiB)": 60.12, + "memory/max_active (GiB)": 46.9, + "memory/max_allocated (GiB)": 46.9, + "ppl": 1.03674, + "step": 8, + "tokens/total": 175713, + "tokens/train_per_sec_per_gpu": 4.47, + "tokens/trainable": 3655 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.178232405622784e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/full_aft_midtrain4/balanced/checkpoint-8/training_args.bin b/full_aft_midtrain4/balanced/checkpoint-8/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0810334748d263e5ac131facb1520f303c2444e6 --- /dev/null +++ b/full_aft_midtrain4/balanced/checkpoint-8/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0107cd88280e968b63f91f7125176eba0c9002c1f534529b1c400b807d0860f +size 8913